{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import checklist\n",
    "import spacy\n",
    "import itertools\n",
    "\n",
    "import checklist.editor\n",
    "import checklist.text_generation\n",
    "from checklist.test_types import MFT, INV, DIR\n",
    "from checklist.expect import Expect\n",
    "from checklist.test_suite import TestSuite\n",
    "import numpy as np\n",
    "import spacy\n",
    "from checklist.perturb import Perturb\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5f707ea14ec14a20b498fff48fa2ada5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=634.0, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "sys.path.append('/home/marcotcr/work/ml-tests/')\n",
    "from mltests import bert_squad_model\n",
    "from checklist.pred_wrapper import PredictorWrapper\n",
    "model = bert_squad_model.BertSquad()\n",
    "invert = lambda a: model.predict_pairs([(x[1], x[0]) for x in a])\n",
    "new_pp = PredictorWrapper.wrap_predict(invert)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "203e1212d9514a15bdaa3da0f175ee6d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['John']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.predict_pairs([('Who is smarter?', 'John is smart')])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "18fffebd963e40f9ae405ba0844de77d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['John']"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "invert([('John is smart', 'Who is smart')])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<checklist.text_generation.TextGenerator at 0x7f17b7351dd8>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "editor = checklist.editor.Editor()\n",
    "editor.tg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp = spacy.load('en_core_web_sm')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def format_squad_with_context(x, pred, conf, label=None, *args, **kwargs):\n",
    "    c, q = x\n",
    "    ret = 'C: %s\\nQ: %s\\n' % (c, q)\n",
    "    if label is not None:\n",
    "        ret += 'A: %s\\n' % label\n",
    "    ret += 'P: %s\\n' % pred\n",
    "    return ret"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def format_squad(x, pred, conf, label=None, *args, **kwargs):\n",
    "    c, q = x\n",
    "    ret = 'Q: %s\\n' % (q)\n",
    "    if label is not None:\n",
    "        ret += 'A: %s\\n' % label\n",
    "    ret += 'P: %s\\n' % pred\n",
    "    return ret"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "def load_squad(fold='validation'):\n",
    "    answers = []\n",
    "    data = []\n",
    "    ids = []\n",
    "    files = {\n",
    "        'validation': '/home/marcotcr/datasets/squad/dev-v1.1.json',\n",
    "        'train': '/home/marcotcr//datasets/squad/train-v1.1.json',\n",
    "        }\n",
    "    f = json.load(open(files[fold]))\n",
    "    for t in f['data']:\n",
    "        for p in t['paragraphs']:\n",
    "            context = p['context']\n",
    "            for qa in p['qas']:\n",
    "                data.append({'passage': context, 'question': qa['question'], 'id': qa['id']})\n",
    "                answers.append(set([(x['text'], x['answer_start']) for x in qa['answers']]))\n",
    "    return data, answers\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "data, answers =  load_squad()\n",
    "spacy_map =  pickle.load(open('/home/marcotcr/tmp/processed_squad.pkl', 'rb'))\n",
    "pairs = [(x['passage'], x['question']) for x in data]\n",
    "processed_pairs = [(spacy_map[x[0]], spacy_map[x[1]]) for x in pairs]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "suite = TestSuite()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Vocabulary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "better, older, smarter, worse, younger, taller, different, more, stronger, bigger, less, shorter, tougher, other, greater, wiser, larger, smaller, faster, richer, cooler, darker, nicer, higher, weaker, happier, longer, hotter, closer, lower, harder, safer, heavier, slower, stranger, easier, quicker, deeper, brighter, simpler, colder, healthier, wealthier, thicker, thinner, cleaner, lighter, quieter, cheaper, poorer, louder, newer, warmer, sharper, wider, lesser, superior, further, earlier, clearer\n"
     ]
    }
   ],
   "source": [
    "print(', '.join(editor.suggest('{first_name} is {mask} than {first_name2}.')[:60]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "adj = ['old', 'smart', 'tall', 'young', 'strong', 'short', 'tough', 'cool', 'fast', 'nice', 'small', 'dark', 'wise', 'rich', 'great', 'weak', 'high', 'slow', 'strange', 'clean']\n",
    "adj = [(x.rstrip('e'), x) for x in adj]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('tall', 'tall')"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "adj[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting 988 examples\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5a8ce55cfc004bec97e76f2e549891ac",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=124.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Test cases:      494\n",
      "Fails (rate):    99 (20.0%)\n",
      "\n",
      "Example fails:\n",
      "C: Jonathan is older than Olivia.\n",
      "Q: Who is less old?\n",
      "A: Olivia\n",
      "P: Jonathan\n",
      "\n",
      "\n",
      "----\n",
      "C: Samantha is slower than Jordan.\n",
      "Q: Who is less slow?\n",
      "A: Jordan\n",
      "P: Samantha is slower than Jordan\n",
      "\n",
      "\n",
      "----\n",
      "C: Kelly is weaker than Ashley.\n",
      "Q: Who is less weak?\n",
      "A: Ashley\n",
      "P: Kelly\n",
      "\n",
      "\n",
      "----\n"
     ]
    }
   ],
   "source": [
    "t = editor.template(\n",
    "    [(\n",
    "    '{first_name} is {adj[0]}er than {first_name1}.',\n",
    "    'Who is less {adj[1]}?'\n",
    "    ),(\n",
    "    '{first_name} is {adj[0]}er than {first_name1}.',\n",
    "    'Who is {adj[0]}er?'\n",
    "    )\n",
    "    ],\n",
    "    labels = ['{first_name1}','{first_name}'],\n",
    "    adj=adj,\n",
    "    remove_duplicates=True,\n",
    "    nsamples=500,\n",
    "    save=True\n",
    "    )\n",
    "name = 'A is COMP than B. Who is more / less COMP?'\n",
    "description = ''\n",
    "test = MFT(**t, name=name, description=description, capability='Vocabulary')\n",
    "test.run(new_pp)\n",
    "test.summary(n=3, format_example_fn=format_squad_with_context)\n",
    "suite.add(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def crossproduct(t):\n",
    "    # takes the output of editor.template and does the cross product of contexts and qas\n",
    "    ret = []\n",
    "    ret_labels = []\n",
    "    for x in t.data:\n",
    "        cs = x['contexts']\n",
    "        qas = x['qas']\n",
    "        d = list(itertools.product(cs, qas))\n",
    "        ret.append([(x[0], x[1][0]) for x in d])\n",
    "        ret_labels.append([x[1][1] for x in d])\n",
    "    t.data = ret\n",
    "    t.labels = ret_labels\n",
    "    return t\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "very, pretty, extremely, quite, also, still, more, really, not, fairly, incredibly, rather, now, generally, already, clearly, relatively, highly, particularly, so, surprisingly, most, currently, certainly, super, definitely, increasingly, being, especially, understandably\n"
     ]
    }
   ],
   "source": [
    "state = editor.suggest('John is very {mask} about the project.')[:20]\n",
    "print(', '.join(editor.suggest('John is {mask} {state} about the project.', state=state)[:30]))\n",
    "very = ['very', 'extremely', 'really', 'quite', 'incredibly', 'particularly', 'highly', 'super']\n",
    "somewhat = ['a little', 'somewhat', 'slightly', 'mildly']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting 5964 examples\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d25d4e525edc4ab8b3bc976bd992d72c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=746.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Test cases:      497\n",
      "Fails (rate):    454 (91.3%)\n",
      "\n",
      "Example fails:\n",
      "C: John is happy about the project. Maria is very happy about the project.\n",
      "Q: Who is least happy about the project?\n",
      "A: John\n",
      "P: Maria\n",
      "\n",
      "\n",
      "----\n",
      "C: Mark is vocal about the project. Danielle is super vocal about the project.\n",
      "Q: Who is most vocal about the project?\n",
      "A: Danielle\n",
      "P: Mark\n",
      "\n",
      "C: Mark is vocal about the project. Danielle is super vocal about the project.\n",
      "Q: Who is least vocal about the project?\n",
      "A: Mark\n",
      "P: Danielle\n",
      "\n",
      "\n",
      "----\n",
      "C: Justin is excited about the project. Brian is super excited about the project.\n",
      "Q: Who is most excited about the project?\n",
      "A: Brian\n",
      "P: Justin\n",
      "\n",
      "C: Justin is excited about the project. Brian is super excited about the project.\n",
      "Q: Who is least excited about the project?\n",
      "A: Justin\n",
      "P: Brian\n",
      "\n",
      "\n",
      "----\n"
     ]
    }
   ],
   "source": [
    "t = crossproduct(editor.template(\n",
    "    {\n",
    "        'contexts': [\n",
    "            '{first_name} is {very} {s} about the project. {first_name1} is {s} about the project.',\n",
    "            '{first_name1} is {s} about the project. {first_name} is {very} {s} about the project.',\n",
    "            '{first_name} is {s} about the project. {first_name1} is {somewhat} {s} about the project.',\n",
    "            '{first_name1} is {somewhat} {s} about the project. {first_name} is {s} about the project.',\n",
    "            '{first_name} is {very} {s} about the project. {first_name1} is {somewhat} {s} about the project.',\n",
    "            '{first_name1} is {somewhat} {s} about the project. {first_name} is {very} {s} about the project.',\n",
    "        ],\n",
    "        'qas': [\n",
    "            (\n",
    "                'Who is most {s} about the project?',\n",
    "                '{first_name}'\n",
    "            ), \n",
    "            (\n",
    "                'Who is least {s} about the project?',\n",
    "                '{first_name1}'\n",
    "            ), \n",
    "            \n",
    "        ]\n",
    "        \n",
    "    },\n",
    "    s = state,\n",
    "    very=very,\n",
    "    somewhat=somewhat,\n",
    "    remove_duplicates=True,\n",
    "    nsamples=500,\n",
    "    save=True\n",
    "    ))\n",
    "name = 'Intensifiers (very, super, extremely) and reducers (somewhat, kinda, etc)?'\n",
    "desc = ''\n",
    "test = MFT(**t, name=name, description=desc, capability='Vocabulary')\n",
    "test.run(new_pp)\n",
    "test.summary(n=3, format_example_fn=format_squad_with_context)\n",
    "suite.add(test)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Taxonomy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Size, chape, color, age, material"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "import munch\n",
    "order = ['size', 'shape', 'age', 'color']\n",
    "props = []\n",
    "properties = {\n",
    "    'color' : ['red', 'blue','yellow', 'green', 'pink', 'white', 'black', 'orange', 'grey', 'purple', 'brown'],\n",
    "    'size' : ['big', 'small', 'tiny', 'enormous'],\n",
    "    'age' : ['old', 'new'],\n",
    "    'shape' : ['round', 'oval', 'square', 'triangular'],\n",
    "    'material' : ['iron', 'wooden', 'ceramic', 'glass', 'stone']\n",
    "}\n",
    "for i in range(len(order)):\n",
    "    for j in range(i + 1, len(order)):\n",
    "        p1, p2 = order[i], order[j]\n",
    "        for v1, v2 in itertools.product(properties[p1], properties[p2]):\n",
    "            props.append(munch.Munch({\n",
    "                'p1': p1,\n",
    "                'p2': p2,\n",
    "                'v1': v1,\n",
    "                'v2': v2,\n",
    "            }))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "couch, sofa, wall, carpet, chair, table, light, door, clock, lamp, mirror, bed, TV, bar, window, tree, box, desk, painting, fridge, curtain, screen, fan, camera, frame, wallpaper, rug, cabinet, elephant, television\n"
     ]
    }
   ],
   "source": [
    "print(', '.join(editor.suggest('There is {a:p.v1} {p.v2} {mask} in the room.', p=props, verbose=False)[:30]))\n",
    "objects = ['box', 'clock', 'table', 'object', 'toy', 'painting', 'sculpture', 'thing', 'figure']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting 2000 examples\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d17425d2c04a4a329f53c000d308c591",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=250.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Test cases:      500\n",
      "Fails (rate):    412 (82.4%)\n",
      "\n",
      "Example fails:\n",
      "C: There is a box in the room. The box is enormous and round.\n",
      "Q: What size is the box?\n",
      "A: enormous\n",
      "P: enormous and round\n",
      "\n",
      "C: There is a box in the room. The box is enormous and round.\n",
      "Q: What shape is the box?\n",
      "A: round\n",
      "P: enormous and round\n",
      "\n",
      "\n",
      "----\n",
      "C: There is a big yellow clock in the room.\n",
      "Q: What size is the clock?\n",
      "A: big\n",
      "P: big yellow\n",
      "\n",
      "C: There is a clock in the room. The clock is big and yellow.\n",
      "Q: What size is the clock?\n",
      "A: big\n",
      "P: big and yellow\n",
      "\n",
      "\n",
      "----\n",
      "C: There is a painting in the room. The painting is round and black.\n",
      "Q: What shape is the painting?\n",
      "A: round\n",
      "P: round and black\n",
      "\n",
      "\n",
      "----\n"
     ]
    }
   ],
   "source": [
    "t = crossproduct(editor.template(\n",
    "    {\n",
    "        'contexts': [\n",
    "            'There is {a:p.v1} {p.v2} {obj} in the room.',\n",
    "            'There is {a:obj} in the room. The {obj} is {p.v1} and {p.v2}.',\n",
    "        ],\n",
    "        'qas': [\n",
    "            (\n",
    "                'What {p.p1} is the {obj}?',\n",
    "                '{p.v1}'\n",
    "            ), \n",
    "            (\n",
    "                'What {p.p2} is the {obj}?',\n",
    "                '{p.v2}'\n",
    "            ), \n",
    "            \n",
    "        ]\n",
    "        \n",
    "    },\n",
    "    obj=objects,\n",
    "    p=props,\n",
    "    remove_duplicates=True,\n",
    "    nsamples=500,\n",
    "    save=True\n",
    "    ))\n",
    "name = 'size, shape, age, color'\n",
    "desc = ''\n",
    "test = MFT(**t, name=name, description=desc, capability='Taxonomy')\n",
    "test.run(new_pp)\n",
    "test.summary(n=3, format_example_fn=format_squad_with_context)\n",
    "suite.add(test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Professions vs nationalities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "professions = editor.suggest('{first_name} works as {a:mask}.')[:30]\n",
    "professions += editor.suggest('{first_name} {last_name} works as {a:mask}.')[:30]\n",
    "professions = list(set(professions))\n",
    "if 'translator' in professions:\n",
    "    professions.remove('translator')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean(string):\n",
    "    return string.lstrip('[a,the,an,in,at] ').rstrip('.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def expect_squad(x, pred, conf, label=None, meta=None):\n",
    "    return clean(pred) == clean(label)\n",
    "expect_squad = Expect.single(expect_squad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting 5000 examples\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "397e42ffad8c4e0187695b186b66ffbc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=625.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Test cases:      500\n",
      "Fails (rate):    247 (49.4%)\n",
      "\n",
      "Example fails:\n",
      "C: Ashley is a Japanese adviser.\n",
      "Q: What is Ashley's job?\n",
      "A: adviser\n",
      "P: Japanese adviser\n",
      "\n",
      "\n",
      "----\n",
      "C: Matthew is a Japanese educator.\n",
      "Q: What is Matthew's job?\n",
      "A: educator\n",
      "P: Japanese educator\n",
      "\n",
      "\n",
      "----\n",
      "C: Timothy is an Indian analyst.\n",
      "Q: What is Timothy's job?\n",
      "A: analyst\n",
      "P: Indian analyst\n",
      "\n",
      "\n",
      "----\n"
     ]
    }
   ],
   "source": [
    "t = crossproduct(editor.template(\n",
    "    {\n",
    "        'contexts': [\n",
    "            '{first_name} is {a:nat} {prof}.',\n",
    "            '{first_name} is {a:prof}. {first_name} is {nat}.',\n",
    "            '{first_name} is {nat}. {first_name} is {a:prof}.',\n",
    "            '{first_name} is {nat} and {a:prof}.',\n",
    "            '{first_name} is {a:prof} and {nat}.',\n",
    "        ],\n",
    "        'qas': [\n",
    "            (\n",
    "                'What is {first_name}\\'s job?',\n",
    "                '{prof}'\n",
    "            ), \n",
    "            (\n",
    "                'What is {first_name}\\'s nationality?',\n",
    "                '{nat}'\n",
    "            ), \n",
    "            \n",
    "        ]\n",
    "        \n",
    "    },\n",
    "    nat = editor.lexicons['nationality'][:10],\n",
    "    prof=professions,\n",
    "    remove_duplicates=True,\n",
    "    nsamples=500,\n",
    "    save=True,\n",
    "    ))\n",
    "name = 'Profession vs nationality'\n",
    "test = MFT(**t, name=name, expect=expect_squad, description='',  capability='Taxonomy')\n",
    "test.run(new_pp)\n",
    "test.summary(n=3, format_example_fn=format_squad_with_context)\n",
    "suite.add(test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Animal vs vehicle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting 2000 examples\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c09ea06a073348c6ad7b4d2be34c6131",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=250.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Test cases:      500\n",
      "Fails (rate):    128 (25.6%)\n",
      "\n",
      "Example fails:\n",
      "C: Austin has a cow and a tractor.\n",
      "Q: What vehicle does Austin have?\n",
      "A: tractor\n",
      "P: a cow and a tractor\n",
      "\n",
      "C: Austin has a tractor and a cow.\n",
      "Q: What animal does Austin have?\n",
      "A: cow\n",
      "P: a tractor and a cow\n",
      "\n",
      "\n",
      "----\n",
      "C: Alexander has a lizard and a firetruck.\n",
      "Q: What animal does Alexander have?\n",
      "A: lizard\n",
      "P: a lizard and a firetruck\n",
      "\n",
      "C: Alexander has a firetruck and a lizard.\n",
      "Q: What animal does Alexander have?\n",
      "A: lizard\n",
      "P: a firetruck and a lizard\n",
      "\n",
      "\n",
      "----\n",
      "C: Angela has a bull and a tractor.\n",
      "Q: What animal does Angela have?\n",
      "A: bull\n",
      "P: a bull and a tractor\n",
      "\n",
      "C: Angela has a bull and a tractor.\n",
      "Q: What vehicle does Angela have?\n",
      "A: tractor\n",
      "P: a bull and a tractor\n",
      "\n",
      "C: Angela has a tractor and a bull.\n",
      "Q: What animal does Angela have?\n",
      "A: bull\n",
      "P: a tractor and a bull\n",
      "\n",
      "\n",
      "----\n"
     ]
    }
   ],
   "source": [
    "animals = ['dog', 'cat', 'bull', 'cow', 'fish', 'serpent', 'snake', 'lizard', 'hamster', 'rabbit', 'guinea pig', 'iguana', 'duck']\n",
    "vehicles = ['car', 'truck', 'train', 'motorcycle', 'bike', 'firetruck', 'tractor', 'van', 'SUV', 'minivan']\n",
    "t = crossproduct(editor.template(\n",
    "    {\n",
    "        'contexts': [\n",
    "            '{first_name} has {a:animal} and {a:vehicle}.',\n",
    "            '{first_name} has {a:vehicle} and {a:animal}.',\n",
    "        ],\n",
    "        'qas': [\n",
    "            (\n",
    "                'What animal does {first_name} have?',\n",
    "                '{animal}'\n",
    "            ), \n",
    "            (\n",
    "                'What vehicle does {first_name} have?',\n",
    "                '{vehicle}'\n",
    "            ), \n",
    "            \n",
    "        ]\n",
    "        \n",
    "    },\n",
    "    animal=animals,\n",
    "    vehicle=vehicles,\n",
    "    remove_duplicates=True,\n",
    "    nsamples=500,\n",
    "    save=True\n",
    "    ))\n",
    "name = 'Animal vs Vehicle'\n",
    "test = MFT(**t, name=name, description='', capability='Taxonomy', expect=expect_squad)\n",
    "test.run(new_pp)\n",
    "test.summary(n=3, format_example_fn=format_squad_with_context)\n",
    "suite.add(test, overwrite=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting 1984 examples\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "df749e1ef8094c17af78a9ce843811d3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=248.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Test cases:      496\n",
      "Fails (rate):    130 (26.2%)\n",
      "\n",
      "Example fails:\n",
      "C: David bought a train. Kayla bought a guinea pig.\n",
      "Q: Who bought an animal?\n",
      "A: Kayla\n",
      "P: David\n",
      "\n",
      "\n",
      "----\n",
      "C: David bought a bull. Mary bought a train.\n",
      "Q: Who bought a vehicle?\n",
      "A: Mary\n",
      "P: David\n",
      "\n",
      "C: Mary bought a train. David bought a bull.\n",
      "Q: Who bought a vehicle?\n",
      "A: Mary\n",
      "P: Mary bought a train. David\n",
      "\n",
      "\n",
      "----\n",
      "C: Christina bought a tractor. Mary bought a cow.\n",
      "Q: Who bought an animal?\n",
      "A: Mary\n",
      "P: Christina\n",
      "\n",
      "\n",
      "----\n"
     ]
    }
   ],
   "source": [
    "animals = ['dog', 'cat', 'bull', 'cow', 'fish', 'serpent', 'snake', 'lizard', 'hamster', 'rabbit', 'guinea pig', 'iguana', 'duck']\n",
    "vehicles = ['car', 'truck', 'train', 'motorcycle', 'bike', 'firetruck', 'tractor', 'van', 'SUV', 'minivan']\n",
    "t = crossproduct(editor.template(\n",
    "    {\n",
    "        'contexts': [\n",
    "            '{first_name} bought {a:animal}. {first_name2} bought {a:vehicle}.',\n",
    "            '{first_name2} bought {a:vehicle}. {first_name} bought {a:animal}.',\n",
    "        ],\n",
    "        'qas': [\n",
    "            (\n",
    "                'Who bought an animal?',\n",
    "                '{first_name}'\n",
    "            ), \n",
    "            (\n",
    "                'Who bought a vehicle?',\n",
    "                '{first_name2}'\n",
    "            ), \n",
    "            \n",
    "        ]\n",
    "        \n",
    "    },\n",
    "    animal=animals,\n",
    "    vehicle=vehicles,\n",
    "    remove_duplicates=True,\n",
    "    nsamples=500,\n",
    "    save=True\n",
    "    ))\n",
    "name = 'Animal vs Vehicle v2'\n",
    "test = MFT(**t, name=name, description='', capability='Taxonomy', expect=expect_squad)\n",
    "test.run(new_pp)\n",
    "test.summary(n=3, format_example_fn=format_squad_with_context)\n",
    "suite.add(test, overwrite=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting 1788 examples\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f7d338adb13944e294dd318add21a650",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=224.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Test cases:      447\n",
      "Fails (rate):    0 (0.0%)\n"
     ]
    }
   ],
   "source": [
    "synonyms = [ ('spiritual', 'religious'), ('angry', 'furious'), ('organized', 'organised'),\n",
    "            ('vocal', 'outspoken'), ('grateful', 'thankful'), ('intelligent', 'smart'),\n",
    "            ('humble', 'modest'), ('courageous', 'brave'), ('happy', 'joyful'), ('scared', 'frightened'),\n",
    "           ]\n",
    "\n",
    "t = crossproduct(editor.template(\n",
    "    {\n",
    "        'contexts': [\n",
    "            '{first_name} is very {s1[0]}. {first_name2} is very {s2[0]}.',\n",
    "            '{first_name2} is very {s2[0]}. {first_name} is very {s1[0]}.',\n",
    "        ],\n",
    "        'qas': [\n",
    "            (\n",
    "                'Who is {s1[1]}?',\n",
    "                '{first_name}'\n",
    "            ), \n",
    "            (\n",
    "                'Who is {s2[1]}?',\n",
    "                '{first_name2}'\n",
    "            ), \n",
    "            \n",
    "        ]\n",
    "        \n",
    "    },\n",
    "    s=synonyms,\n",
    "    remove_duplicates=True,\n",
    "    nsamples=250,\n",
    "    save=True\n",
    "   ))\n",
    "t += crossproduct(editor.template(\n",
    "    {\n",
    "        'contexts': [\n",
    "            '{first_name} is very {s1[1]}. {first_name2} is very {s2[1]}.',\n",
    "            '{first_name2} is very {s2[1]}. {first_name} is very {s1[1]}.',\n",
    "        ],\n",
    "        'qas': [\n",
    "            (\n",
    "                'Who is {s1[0]}?',\n",
    "                '{first_name}'\n",
    "            ), \n",
    "            (\n",
    "                'Who is {s2[0]}?',\n",
    "                '{first_name2}'\n",
    "            ), \n",
    "            \n",
    "        ]\n",
    "        \n",
    "    },\n",
    "    s=synonyms,\n",
    "    remove_duplicates=True,\n",
    "    nsamples=250,\n",
    "    save=True\n",
    "    )) \n",
    "name = 'Synonyms'\n",
    "test = MFT(**t, name=name, description='', capability='Taxonomy', expect=expect_squad)\n",
    "test.run(new_pp)\n",
    "test.summary(n=3, format_example_fn=format_squad_with_context)\n",
    "suite.add(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "comp_pairs = [('better', 'worse'), ('older', 'younger'), ('smarter', 'dumber'), ('taller', 'shorter'), ('bigger', 'smaller'), ('stronger', 'weaker'), ('faster', 'slower'), ('darker', 'lighter'), ('richer', 'poorer'), ('happier', 'sadder'), ('louder', 'quieter'), ('warmer', 'colder')]\n",
    "comp_pairs = list(set(comp_pairs))#list(set(comp_pairs + [(x[1], x[0]) for x in comp_pairs]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting 1984 examples\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7ee7d77ac27d49cd9a33c34b22d4f01a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=248.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Test cases:      496\n",
      "Fails (rate):    334 (67.3%)\n",
      "\n",
      "Example fails:\n",
      "C: Jacob is colder than Matthew.\n",
      "Q: Who is warmer?\n",
      "A: Matthew\n",
      "P: Jacob\n",
      "\n",
      "\n",
      "----\n",
      "C: Ryan is older than Amber.\n",
      "Q: Who is younger?\n",
      "A: Amber\n",
      "P: Ryan\n",
      "\n",
      "\n",
      "----\n",
      "C: Andrea is quieter than Maria.\n",
      "Q: Who is louder?\n",
      "A: Maria\n",
      "P: Andrea\n",
      "\n",
      "\n",
      "----\n"
     ]
    }
   ],
   "source": [
    "t = crossproduct(editor.template(\n",
    "    {\n",
    "        'contexts': [\n",
    "            '{first_name} is {comp[0]} than {first_name1}.',\n",
    "            '{first_name1} is {comp[1]} than {first_name}.',\n",
    "        ],\n",
    "        'qas': [\n",
    "            (\n",
    "                'Who is {comp[1]}?',\n",
    "                '{first_name1}',\n",
    "            ),\n",
    "            (\n",
    "                'Who is {comp[0]}?',\n",
    "                '{first_name}',\n",
    "            )\n",
    "            \n",
    "        ]\n",
    "        ,\n",
    "    },\n",
    "    comp=comp_pairs,\n",
    "    remove_duplicates=True,\n",
    "    nsamples=500,\n",
    "    save=True\n",
    "    ))\n",
    "name = 'A is COMP than B. Who is antonym(COMP)? B'\n",
    "test = MFT(**t, name=name, description='', capability='Taxonomy')\n",
    "test.run(new_pp)\n",
    "test.summary(n=3, format_example_fn=format_squad_with_context)\n",
    "suite.add(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting 7856 examples\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6aa7a8dc05674909b576bbaebb25733c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=982.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Test cases:      491\n",
      "Fails (rate):    491 (100.0%)\n",
      "\n",
      "Example fails:\n",
      "C: Heather is more unhappy than Rachel.\n",
      "Q: Who is more happy?\n",
      "A: Rachel\n",
      "P: Heather\n",
      "\n",
      "C: Rachel is more happy than Heather.\n",
      "Q: Who is more unhappy?\n",
      "A: Heather\n",
      "P: Rachel\n",
      "\n",
      "C: Rachel is more happy than Heather.\n",
      "Q: Who is less unhappy?\n",
      "A: Rachel\n",
      "P: Heather\n",
      "\n",
      "\n",
      "----\n",
      "C: Michelle is more religious than Jordan.\n",
      "Q: Who is more secular?\n",
      "A: Jordan\n",
      "P: Michelle\n",
      "\n",
      "C: Michelle is more religious than Jordan.\n",
      "Q: Who is less secular?\n",
      "A: Michelle\n",
      "P: Jordan\n",
      "\n",
      "C: Jordan is more secular than Michelle.\n",
      "Q: Who is more religious?\n",
      "A: Michelle\n",
      "P: Jordan\n",
      "\n",
      "\n",
      "----\n",
      "C: Elizabeth is more cautious than Timothy.\n",
      "Q: Who is more brave?\n",
      "A: Timothy\n",
      "P: Elizabeth\n",
      "\n",
      "C: Elizabeth is more cautious than Timothy.\n",
      "Q: Who is less brave?\n",
      "A: Elizabeth\n",
      "P: Timothy\n",
      "\n",
      "C: Timothy is more brave than Elizabeth.\n",
      "Q: Who is more cautious?\n",
      "A: Elizabeth\n",
      "P: Timothy\n",
      "\n",
      "\n",
      "----\n"
     ]
    }
   ],
   "source": [
    "antonym_adjs = [('progressive', 'conservative'),('religious', 'secular'),('positive', 'negative'),('defensive', 'offensive'),('rude',  'polite'),('optimistic', 'pessimistic'),('stupid', 'smart'),('negative', 'positive'),('unhappy', 'happy'),('active', 'passive'),('impatient', 'patient'),('powerless', 'powerful'),('visible', 'invisible'),('fat', 'thin'),('bad', 'good'),('cautious', 'brave'), ('hopeful', 'hopeless'),('insecure', 'secure'),('humble', 'proud'),('passive', 'active'),('dependent', 'independent'),('pessimistic', 'optimistic'),('irresponsible', 'responsible'),('courageous', 'fearful')]\n",
    "t = crossproduct(editor.template(\n",
    "    {\n",
    "        'contexts': [\n",
    "            '{first_name} is more {a[0]} than {first_name1}.',\n",
    "            '{first_name1} is more {a[1]} than {first_name}.',\n",
    "            '{first_name} is less {a[1]} than {first_name1}.',\n",
    "            '{first_name1} is less {a[0]} than {first_name}.',\n",
    "        ],\n",
    "        'qas': [\n",
    "            (\n",
    "                'Who is more {a[0]}?',\n",
    "                '{first_name}',\n",
    "            ),\n",
    "            (\n",
    "                'Who is less {a[0]}?',\n",
    "                '{first_name1}',\n",
    "            ),\n",
    "            (\n",
    "                'Who is more {a[1]}?',\n",
    "                '{first_name1}',\n",
    "            ),\n",
    "            (\n",
    "                'Who is less {a[1]}?',\n",
    "                '{first_name}',\n",
    "            ),\n",
    "        ]\n",
    "        ,\n",
    "    },\n",
    "    a = antonym_adjs,\n",
    "    remove_duplicates=True,\n",
    "    nsamples=500,\n",
    "    save=True\n",
    "    ))\n",
    "name = 'A is more X than B. Who is more antonym(X)? B. Who is less X? B. Who is more X? A. Who is less antonym(X)? A.'\n",
    "test = MFT(**t, name=name, description='', capability='Taxonomy')\n",
    "test.run(new_pp)\n",
    "test.summary(n=3, format_example_fn=format_squad_with_context)\n",
    "suite.add(test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Robustness"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "typos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting 1000 examples\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b209075092494c9b87186fe07ed586d3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=130.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Test cases:      500\n",
      "Fails (rate):    58 (11.6%)\n",
      "\n",
      "Example fails:\n",
      "Q: What is the theory that this King's name is the origin of \"Huguenot\" called?\n",
      "P: Hugues hypothesis\n",
      "\n",
      "Q: What is the theory that this King's name is the origin of \"Huguenot\" calle?d\n",
      "P: The \"Hugues hypothesis\n",
      "\n",
      "\n",
      "----\n",
      "Q: Who did the Broncos beat to win their division in 2015?\n",
      "P: Pittsburgh Steelers\n",
      "\n",
      "Q: Who did the Broncos beat to win theird ivision in 2015?\n",
      "P: New England Patriots\n",
      "\n",
      "\n",
      "----\n",
      "Q: How did Luther describe his learning at the university?\n",
      "P: rote learning\n",
      "\n",
      "Q: How did Luther describe his leanring at the university?\n",
      "P: a beerhouse and whorehouse\n",
      "\n",
      "\n",
      "----\n"
     ]
    }
   ],
   "source": [
    "def question_typo(x):\n",
    "    return (x[0], Perturb.add_typos(x[1]))\n",
    "t = Perturb.perturb(pairs, question_typo, nsamples=500)\n",
    "test = INV(**t, name='Question typo', capability='Robustness', description='')\n",
    "test.run(new_pp)\n",
    "test.summary(n=3, format_example_fn=format_squad)\n",
    "suite.add(test, overwrite=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Contractions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting 1005 examples\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3ee4e206bb5744f4b1bf733f9d6aaadb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=129.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Test cases:      500\n",
      "Fails (rate):    17 (3.4%)\n",
      "\n",
      "Example fails:\n",
      "Q: What is a connection identifier \n",
      "P: packets include a connection identifier rather than address information\n",
      "\n",
      "Q: What's a connection identifier \n",
      "P: The packets include a connection identifier rather than address information\n",
      "\n",
      "\n",
      "----\n",
      "Q: Where did the Meuse flow before the flood? \n",
      "P: just south of today's line Merwede-Oude Maas\n",
      "\n",
      "Q: Where'd the Meuse flow before the flood? \n",
      "P: just south of today's line Merwede-Oude Maas to the North Sea\n",
      "\n",
      "\n",
      "----\n",
      "Q: What do red algal chloroplasts have that green chloroplasts don't?\n",
      "P: phycobilisomes\n",
      "\n",
      "Q: What do red algal chloroplasts have that green chloroplasts do not?\n",
      "P: chlorophyll b\n",
      "\n",
      "\n",
      "----\n"
     ]
    }
   ],
   "source": [
    "def contractions(x):\n",
    "    conts = Perturb.contractions(x[1])\n",
    "    return [(x[0], a) for a in conts]\n",
    "t = Perturb.perturb(pairs, contractions, nsamples=500)\n",
    "test = INV(**t, name='Question contractions', capability='Robustness', description='')\n",
    "test.run(new_pp)\n",
    "test.summary(n=3, format_example_fn=format_squad)\n",
    "suite.add(test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Add random sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "random_sentences = set()\n",
    "for x, _ in processed_pairs:\n",
    "    for y in x.sents:\n",
    "        random_sentences.add(y.text)\n",
    "random_sentences = list(random_sentences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "# len(random_sentences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting 1500 examples\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5b5898e4bc0b466295f5f858b72727cc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=194.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Test cases:      500\n",
      "Fails (rate):    49 (9.8%)\n",
      "\n",
      "Example fails:\n",
      "Q: What is the main reason consulting pharmacists are increasingly working directly with patients?\n",
      "P: many elderly people are now taking numerous medications\n",
      "\n",
      "Q: What is the main reason consulting pharmacists are increasingly working directly with patients?\n",
      "P: many elderly people are now taking numerous medications but continue to live outside of institutional settings\n",
      "Perturb: add to beg: Kublai botched his campaigns against Annam, Champa, and Java, but won a Pyrrhic victory against Burma. \n",
      "\n",
      "\n",
      "----\n",
      "Q: The adaptive immune system must distinguish between what types of molecules?\n",
      "P: self and non-self molecules\n",
      "\n",
      "Q: The adaptive immune system must distinguish between what types of molecules?\n",
      "P: self and non-self\n",
      "Perturb: add to beg: Many churches preserve sculptured fonts, capitals, and more importantly mosaics, which were common in Norman Italy and drew heavily on the Greek heritage. \n",
      "\n",
      "\n",
      "----\n",
      "Q: What did these flights test on the CM?\n",
      "P: the Service Module engine and the Command Module heat shield\n",
      "\n",
      "Q: What did these flights test on the CM?\n",
      "P: Service Module engine and the Command Module heat shield\n",
      "Perturb: add to end: The same report found that 3% of the surveyed audience regarded the show as \"very unsuitable\" for family viewing. \n",
      "\n",
      "Q: What did these flights test on the CM?\n",
      "P: Service Module engine and the Command Module heat shield\n",
      "Perturb: add to beg: The same report found that 3% of the surveyed audience regarded the show as \"very unsuitable\" for family viewing. \n",
      "\n",
      "\n",
      "----\n"
     ]
    }
   ],
   "source": [
    "def add_random_sentence(x, **kwargs):\n",
    "    random_s = np.random.choice(random_sentences)\n",
    "    while random_s in x[0]:\n",
    "        random_s = np.random.choice(random_sentences)\n",
    "    random_s = random_s.strip('.') + '. '\n",
    "    meta = ['add to end: %s' % random_s, 'add to beg: %s' % random_s]\n",
    "    return [(x[0] + random_s, x[1]), (random_s + x[0], x[1])], meta\n",
    "\n",
    "def format_add(x, pred, conf, label=None, meta=None):\n",
    "    ret = format_squad(x, pred, conf, label, meta)\n",
    "    if meta:\n",
    "        ret += 'Perturb: %s\\n' % meta\n",
    "    return ret\n",
    "\n",
    "t = Perturb.perturb(pairs, add_random_sentence, nsamples=500, meta=True)\n",
    "test = INV(**t, name='Add random sentence to context', capability='Robustness', description='')\n",
    "test.run(new_pp)\n",
    "test.summary(n=3, format_example_fn=format_add)\n",
    "suite.add(test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## NER"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "def change_thing(change_fn):\n",
    "    def change_both(cq, **kwargs):\n",
    "        context, question = cq\n",
    "        a = change_fn(context, meta=True)\n",
    "        if not a:\n",
    "            return None\n",
    "        changed, meta = a\n",
    "        ret = []\n",
    "        for c, m in zip(changed, meta):\n",
    "            new_q = re.sub(r'\\b%s\\b' % re.escape(m[0]), m[1], question.text)\n",
    "            ret.append((c, new_q))\n",
    "        return ret, meta\n",
    "    return change_both\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "def expect_same(orig_pred, pred, orig_conf, conf, labels=None, meta=None):\n",
    "    if not meta:\n",
    "        return pred == orig_pred\n",
    "    return pred == re.sub(r'\\b%s\\b' % re.escape(meta[0]), meta[1], orig_pred)\n",
    "\n",
    "def format_replace(x, pred, conf, label=None, meta=None):\n",
    "    ret = format_squad(x, pred, conf, label, meta)\n",
    "    if meta:\n",
    "        ret += 'Perturb: %s -> %s\\n' % meta\n",
    "    return ret\n",
    "\n",
    "def format_replace_context(x, pred, conf, label=None, meta=None):\n",
    "    ret = format_squad_with_context(x, pred, conf, label, meta)\n",
    "    if meta:\n",
    "        ret += 'Perturb: %s -> %s\\n' % meta\n",
    "    return ret"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting 5500 examples\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1a8de75bac6940b4851bdd78acd4f21e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=713.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Test cases:      500\n",
      "Fails (rate):    29 (5.8%)\n",
      "\n",
      "Example fails:\n",
      "Q: What American actor is also a university graduate?\n",
      "P: Philip Kaufman\n",
      "\n",
      "Q: What American actor is also a university graduate?\n",
      "P: Ed Asner\n",
      "Perturb: Carl Van Vechten -> David Rivera\n",
      "\n",
      "Q: What American actor is also a university graduate?\n",
      "P: Ed Asner\n",
      "Perturb: Carl Van Vechten -> Daniel Campbell\n",
      "\n",
      "\n",
      "----\n",
      "Q: The Codex Forster is a collection of notebooks by which famous Italian Renaissance polymath?\n",
      "P: Leonardo da Vinci\n",
      "\n",
      "Q: The Codex Forster is a collection of notebooks by which famous Italian Renaissance polymath?\n",
      "P: Michael Myers\n",
      "Perturb: Leonardo da Vinci's -> Michael Myers\n",
      "\n",
      "Q: The Codex Forster is a collection of notebooks by which famous Italian Renaissance polymath?\n",
      "P: Christopher Miller\n",
      "Perturb: Leonardo da Vinci's -> Christopher Miller\n",
      "\n",
      "\n",
      "----\n",
      "Q: Who did Luther view to be the Antichrist?\n",
      "P: the papacy, and the Roman Church\n",
      "\n",
      "Q: Who did Adam view to be the Antichrist?\n",
      "P: the papacy\n",
      "Perturb: Luther -> Adam\n",
      "\n",
      "\n",
      "----\n"
     ]
    }
   ],
   "source": [
    "t = Perturb.perturb(processed_pairs, change_thing(Perturb.change_names), nsamples=500, meta=True)\n",
    "\n",
    "test = INV(**t, name='Change name everywhere', capability='NER',\n",
    "          description='', expect=Expect.pairwise(expect_same))\n",
    "test.run(new_pp)\n",
    "test.summary(3, format_example_fn=format_replace)\n",
    "suite.add(test, overwrite=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting 5500 examples\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "999366cdf6f04784ae09f995a6ca544f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=725.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Test cases:      500\n",
      "Fails (rate):    37 (7.4%)\n",
      "\n",
      "Example fails:\n",
      "Q: Who was sacked as the first half clock expired?\n",
      "P: Newton\n",
      "\n",
      "Q: Who was sacked as the first half clock expired?\n",
      "P: DeMarcus Ware\n",
      "Perturb: Newton -> Buffalo\n",
      "\n",
      "Q: Who was sacked as the first half clock expired?\n",
      "P: DeMarcus Ware\n",
      "Perturb: Newton -> Minneapolis\n",
      "\n",
      "\n",
      "----\n",
      "Q: What is an example of an article of uniform clothing typically present in Australian private schools?\n",
      "P: compulsory blazer\n",
      "\n",
      "Q: What is an example of an article of uniform clothing typically present in Australian private schools?\n",
      "P: a compulsory blazer\n",
      "Perturb: Australia -> Peru\n",
      "\n",
      "\n",
      "----\n",
      "Q: Ludwig Krapf recorded the name was what?\n",
      "P: both Kenia and Kegnia\n",
      "\n",
      "Q: Ludwig Krapf recorded the name was what?\n",
      "P: Kenia and Kegnia\n",
      "Perturb: Kenya -> Philippines\n",
      "\n",
      "Q: Ludwig Krapf recorded the name was what?\n",
      "P: Kenia and Kegnia\n",
      "Perturb: Kenya -> Morocco\n",
      "\n",
      "\n",
      "----\n"
     ]
    }
   ],
   "source": [
    "t = Perturb.perturb(processed_pairs, change_thing(Perturb.change_location), nsamples=500, meta=True)\n",
    "\n",
    "test = INV(**t, name='Change location everywhere', capability='NER',\n",
    "          description='', expect=Expect.pairwise(expect_same))\n",
    "test.run(new_pp)\n",
    "test.summary(3, format_example_fn=format_replace)\n",
    "suite.add(test, overwrite=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Temporal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting 964 examples\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1510c4bfba0f401e90f12c1c85ed3986",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=121.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Test cases:      482\n",
      "Fails (rate):    200 (41.5%)\n",
      "\n",
      "Example fails:\n",
      "C: Both Aaron and Mark were photographers, but there was a change in Aaron, who is now a producer.\n",
      "Q: Who is a producer?\n",
      "A: Aaron\n",
      "P: Aaron and Mark were photographers, but there was a change in Aaron\n",
      "\n",
      "\n",
      "----\n",
      "C: Both Brian and David were educators, but there was a change in David, who is now an artist.\n",
      "Q: Who is an artist?\n",
      "A: David\n",
      "P: David were educators, but there was a change in David\n",
      "\n",
      "\n",
      "----\n",
      "C: Both Megan and Aaron were artists, but there was a change in Aaron, who is now an architect.\n",
      "Q: Who is an architect?\n",
      "A: Aaron\n",
      "P: Aaron were artists, but there was a change in Aaron\n",
      "\n",
      "\n",
      "----\n"
     ]
    }
   ],
   "source": [
    "t = crossproduct(editor.template(\n",
    "    {\n",
    "        'contexts': [\n",
    "            'Both {first_name} and {first_name2} were {prof1}s, but there was a change in {first_name}, who is now {a:prof2}.',\n",
    "            'Both {first_name2} and {first_name} were {prof1}s, but there was a change in {first_name}, who is now {a:prof2}.',\n",
    "        ],\n",
    "        'qas': [\n",
    "            (\n",
    "                'Who is {a:prof2}?',\n",
    "                '{first_name}'\n",
    "            ), \n",
    "        ]\n",
    "        \n",
    "    },\n",
    "    save=True,\n",
    "    prof=professions,\n",
    "    remove_duplicates=True,\n",
    "    nsamples=500,\n",
    "    ))\n",
    "name = 'There was a change in profession'\n",
    "test = MFT(**t, expect=expect_squad, capability='Temporal', name=name, description='' )\n",
    "test.run(new_pp)\n",
    "test.summary(n=3, format_example_fn=format_squad_with_context)\n",
    "suite.add(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting 1984 examples\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "44db5d158cd8420ea0b656abea3fa227",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=248.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Test cases:      496\n",
      "Fails (rate):    411 (82.9%)\n",
      "\n",
      "Example fails:\n",
      "C: Amber became a economist before Daniel did.\n",
      "Q: Who became a economist last?\n",
      "A: Daniel\n",
      "P: Amber\n",
      "\n",
      "\n",
      "----\n",
      "C: Emily became a educator after Amy did.\n",
      "Q: Who became a educator first?\n",
      "A: Amy\n",
      "P: Emily\n",
      "\n",
      "C: Emily became a educator after Amy did.\n",
      "Q: Who became a educator last?\n",
      "A: Emily\n",
      "P: Amy\n",
      "\n",
      "\n",
      "----\n",
      "C: Samuel became a academic after Jacob did.\n",
      "Q: Who became a academic first?\n",
      "A: Jacob\n",
      "P: Samuel\n",
      "\n",
      "\n",
      "----\n"
     ]
    }
   ],
   "source": [
    "t = crossproduct(editor.template(\n",
    "    {\n",
    "        'contexts': [\n",
    "            '{first_name} became a {prof} before {first_name2} did.',\n",
    "            '{first_name2} became a {prof} after {first_name} did.',\n",
    "        ],\n",
    "        'qas': [\n",
    "            (\n",
    "                'Who became a {prof} first?',\n",
    "                '{first_name}'\n",
    "            ), \n",
    "            (\n",
    "                'Who became a {prof} last?',\n",
    "                '{first_name2}'\n",
    "            ), \n",
    "        ]\n",
    "        \n",
    "    },\n",
    "    save=True,\n",
    "    prof=professions,\n",
    "    remove_duplicates=True,\n",
    "    nsamples=500,\n",
    "    ))\n",
    "name = 'Understanding before / after -> first / last.'\n",
    "test = MFT(**t, expect=expect_squad, capability='Temporal', name=name, description='' )\n",
    "test.run(new_pp)\n",
    "test.summary(n=3, format_example_fn=format_squad_with_context)\n",
    "suite.add(test)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Negation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In context"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting 1996 examples\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e7db5059d640419cb43d75be86aa167a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=250.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Test cases:      499\n",
      "Fails (rate):    337 (67.5%)\n",
      "\n",
      "Example fails:\n",
      "C: Jason is an intern. Emily is not.\n",
      "Q: Who is not an intern?\n",
      "A: Emily\n",
      "P: Jason\n",
      "\n",
      "\n",
      "----\n",
      "C: Brandon is not an author. Robert is.\n",
      "Q: Who is an author?\n",
      "A: Robert\n",
      "P: Brandon\n",
      "\n",
      "\n",
      "----\n",
      "C: William is not an architect. Ryan is.\n",
      "Q: Who is an architect?\n",
      "A: Ryan\n",
      "P: William\n",
      "\n",
      "\n",
      "----\n"
     ]
    }
   ],
   "source": [
    "t = crossproduct(editor.template(\n",
    "    {\n",
    "        'contexts': [\n",
    "            '{first_name} is not {a:prof}. {first_name2} is.',\n",
    "            '{first_name2} is {a:prof}. {first_name} is not.',\n",
    "        ],\n",
    "        'qas': [\n",
    "            (\n",
    "                'Who is {a:prof}?',\n",
    "                '{first_name2}'\n",
    "            ), \n",
    "            (\n",
    "                'Who is not {a:prof}?',\n",
    "                '{first_name}'\n",
    "            ), \n",
    "        ]\n",
    "        \n",
    "    },\n",
    "    save=True,\n",
    "    prof=professions,\n",
    "    remove_duplicates=True,\n",
    "    nsamples=500,\n",
    "    ))\n",
    "name = 'Negation in context, may or may not be in question'\n",
    "test = MFT(**t, expect=expect_squad, capability='Negation', name=name, description='' )\n",
    "test.run(new_pp)\n",
    "test.summary(n=3, format_example_fn=format_squad_with_context)\n",
    "suite.add(test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Not in context:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting 3848 examples\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5f2ee3b95e4c4cf8a7c2d978ce3909f0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=481.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Test cases:      481\n",
      "Fails (rate):    481 (100.0%)\n",
      "\n",
      "Example fails:\n",
      "C: Daniel is an actor. Kevin is an analyst.\n",
      "Q: Who is not an actor?\n",
      "A: Kevin\n",
      "P: Daniel\n",
      "\n",
      "C: Daniel is an actor. Kevin is an analyst.\n",
      "Q: Who is not an analyst?\n",
      "A: Daniel\n",
      "P: Kevin\n",
      "\n",
      "C: Kevin is an analyst. Daniel is an actor.\n",
      "Q: Who is not an actor?\n",
      "A: Kevin\n",
      "P: Daniel\n",
      "\n",
      "\n",
      "----\n",
      "C: Amanda is an organizer. Stephanie is an activist.\n",
      "Q: Who is not an organizer?\n",
      "A: Stephanie\n",
      "P: Amanda\n",
      "\n",
      "C: Amanda is an organizer. Stephanie is an activist.\n",
      "Q: Who is not an activist?\n",
      "A: Amanda\n",
      "P: Stephanie\n",
      "\n",
      "C: Stephanie is an activist. Amanda is an organizer.\n",
      "Q: Who is not an organizer?\n",
      "A: Stephanie\n",
      "P: Amanda\n",
      "\n",
      "\n",
      "----\n",
      "C: Jacob is an accountant. Elizabeth is a nurse.\n",
      "Q: Who is not an accountant?\n",
      "A: Elizabeth\n",
      "P: Jacob\n",
      "\n",
      "C: Jacob is an accountant. Elizabeth is a nurse.\n",
      "Q: Who is not a nurse?\n",
      "A: Jacob\n",
      "P: Elizabeth\n",
      "\n",
      "C: Elizabeth is a nurse. Jacob is an accountant.\n",
      "Q: Who is not an accountant?\n",
      "A: Elizabeth\n",
      "P: Jacob\n",
      "\n",
      "\n",
      "----\n"
     ]
    }
   ],
   "source": [
    "\n",
    "t = crossproduct(editor.template(\n",
    "    {\n",
    "        'contexts': [\n",
    "            '{first_name} is {a:prof}. {first_name2} is {a:prof2}.',\n",
    "            '{first_name2} is {a:prof2}. {first_name} is {a:prof}.',\n",
    "        ],\n",
    "        'qas': [\n",
    "            (\n",
    "                'Who is {a:prof}?',\n",
    "                '{first_name}'\n",
    "            ), \n",
    "            (\n",
    "                'Who is not {a:prof}?',\n",
    "                '{first_name2}'\n",
    "            ), \n",
    "            (\n",
    "                'Who is {a:prof2}?',\n",
    "                '{first_name2}'\n",
    "            ), \n",
    "            (\n",
    "                'Who is not {a:prof2}?',\n",
    "                '{first_name}'\n",
    "            ), \n",
    "        ]\n",
    "        \n",
    "    },\n",
    "    prof=professions,\n",
    "    remove_duplicates=True,\n",
    "    nsamples=500,\n",
    "    ))\n",
    "name = 'Negation in question only.'\n",
    "test = MFT(**t, expect=expect_squad, capability='Negation', name=name, description='' )\n",
    "test.run(new_pp)\n",
    "test.summary(n=3, format_example_fn=format_squad_with_context)\n",
    "suite.add(test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fairness spinoff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting 2000 examples\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3d611004e0594c2a9f9bec673293d007",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=250.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "profession    fail_men fail_women (count)\n",
      "CEO             0.17      0.97     (267)\n",
      "doctor          0.03      0.89     (247)\n",
      "secretary       0.60      0.04     (253)\n",
      "nurse           0.58      0.41     (233)\n"
     ]
    }
   ],
   "source": [
    "import collections\n",
    "fewer_profs = ['doctor', 'nurse', 'secretary', 'CEO']\n",
    "t = editor.template(\n",
    "    [\n",
    "        ('{male} is not {a:prof}, {female} is.', 'Who is {a:prof}?', '{female}', 'woman', '{prof}'),\n",
    "        ('{female} is not {a:prof}, {male} is.', 'Who is {a:prof}?', '{male}', 'man', '{prof}'),\n",
    "    ],\n",
    "#     prof=professions + ['doctor'],\n",
    "    prof=fewer_profs,\n",
    "    remove_duplicates=True,\n",
    "    nsamples=1000,\n",
    "    unroll=True,\n",
    "    save=True,\n",
    "    )\n",
    "data = [(d[0], d[1]) for d in t.data]\n",
    "labels = [d[2] for d in t.data]\n",
    "meta = [(d[3], d[4]) for d in t.data]\n",
    "\n",
    "test = MFT(data, expect=expect_squad, labels=labels, meta=meta, templates=t.templates,\n",
    "          name='M/F failure rates should be similar for different professions', capability='Fairness',\n",
    "          description='Using negation in context.')\n",
    "test.run(new_pp)\n",
    "\n",
    "def print_fair(test):\n",
    "    c = collections.Counter(test.meta)\n",
    "    fail = collections.Counter([tuple(x) for x in np.array(test.meta)[test.fail_idxs()]])\n",
    "    profs = set()\n",
    "    for sex, prof in fail:\n",
    "        profs.add(prof)\n",
    "    prof_fail = {}\n",
    "    get_fail = lambda f:fail[f] / c[f]\n",
    "    for prof in profs:\n",
    "        fail_m = get_fail(('man', prof))\n",
    "        fail_f = get_fail(('woman', prof))\n",
    "        prof_fail[prof] = (fail_m, fail_f)\n",
    "    print('%-13s fail_men fail_women (count)' % 'profession')\n",
    "    for prof, vs in sorted(prof_fail.items(), key=lambda x:max(x[1][0], x[1][1]), reverse=True):\n",
    "        fail_m, fail_f = vs\n",
    "        print('%-13s   %.1f      %.1f     (%d)' % (prof, 100 * fail_m, 100 * fail_f, c[('man', prof)]))\n",
    "print_fair(test)\n",
    "suite.add(test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Coref"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Basic coref"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "if 'actress' in professions:\n",
    "    professions.remove('actress')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting 3816 examples\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c5143e4a3a1d4d61b012be725d861b66",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=477.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Test cases:      477\n",
      "Fails (rate):    477 (100.0%)\n",
      "\n",
      "Example fails:\n",
      "C: Katherine and Paul are friends. He is an advisor, and she is an analyst.\n",
      "Q: Who is an analyst?\n",
      "A: Katherine\n",
      "P: Paul\n",
      "\n",
      "C: Paul and Katherine are friends. She is an analyst, and he is an advisor.\n",
      "Q: Who is an advisor?\n",
      "A: Paul\n",
      "P: Katherine\n",
      "\n",
      "C: Katherine and Paul are friends. She is an analyst, and he is an advisor.\n",
      "Q: Who is an analyst?\n",
      "A: Katherine\n",
      "P: Paul\n",
      "\n",
      "\n",
      "----\n",
      "C: Paul and Taylor are friends. He is an organizer, and she is a photographer.\n",
      "Q: Who is an organizer?\n",
      "A: Paul\n",
      "P: Paul and Taylor\n",
      "\n",
      "C: Taylor and Paul are friends. He is an organizer, and she is a photographer.\n",
      "Q: Who is a photographer?\n",
      "A: Taylor\n",
      "P: Paul\n",
      "\n",
      "C: Paul and Taylor are friends. She is a photographer, and he is an organizer.\n",
      "Q: Who is an organizer?\n",
      "A: Paul\n",
      "P: Taylor\n",
      "\n",
      "\n",
      "----\n",
      "C: Rebecca and Gabriel are friends. He is an executive, and she is an author.\n",
      "Q: Who is an author?\n",
      "A: Rebecca\n",
      "P: Gabriel\n",
      "\n",
      "C: Gabriel and Rebecca are friends. She is an author, and he is an executive.\n",
      "Q: Who is an executive?\n",
      "A: Gabriel\n",
      "P: Rebecca\n",
      "\n",
      "C: Rebecca and Gabriel are friends. She is an author, and he is an executive.\n",
      "Q: Who is an author?\n",
      "A: Rebecca\n",
      "P: Gabriel\n",
      "\n",
      "\n",
      "----\n"
     ]
    }
   ],
   "source": [
    "t = crossproduct(editor.template(\n",
    "    {\n",
    "        'contexts': [\n",
    "            '{male} and {female} are friends. He is {a:prof1}, and she is {a:prof2}.',\n",
    "            '{female} and {male} are friends. He is {a:prof1}, and she is {a:prof2}.',\n",
    "            '{male} and {female} are friends. She is {a:prof2}, and he is {a:prof1}.',\n",
    "            '{female} and {male} are friends. She is {a:prof2}, and he is {a:prof1}.',\n",
    "        ],\n",
    "        'qas': [\n",
    "            (\n",
    "                'Who is {a:prof1}?',\n",
    "                '{male}'\n",
    "            ), \n",
    "            (\n",
    "                'Who is {a:prof2}?',\n",
    "                '{female}'\n",
    "            ), \n",
    "        ]\n",
    "        \n",
    "    },\n",
    "    save=True,\n",
    "    prof=professions,\n",
    "    remove_duplicates=True,\n",
    "    nsamples=500,\n",
    "    ))\n",
    "name = 'Basic coref, he / she'\n",
    "test = MFT(**t, expect=expect_squad, name=name, description='', capability='Coref')\n",
    "test.run(new_pp)\n",
    "test.summary(n=3, format_example_fn=format_squad_with_context)\n",
    "suite.add(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting 1000 examples\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "19e702251a8348029c09b9b7b5cc2111",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=125.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Test cases:      500\n",
      "Fails (rate):    459 (91.8%)\n",
      "\n",
      "Example fails:\n",
      "C: Erica and Paul are friends. Her mom is an academic.\n",
      "Q: Whose mom is an academic?\n",
      "A: Erica\n",
      "P: Paul\n",
      "\n",
      "\n",
      "----\n",
      "C: Leah and Chad are friends. Her mom is an advisor.\n",
      "Q: Whose mom is an advisor?\n",
      "A: Leah\n",
      "P: Chad\n",
      "\n",
      "\n",
      "----\n",
      "C: Megan and Nathan are friends. Her mom is an accountant.\n",
      "Q: Whose mom is an accountant?\n",
      "A: Megan\n",
      "P: Megan and Nathan\n",
      "\n",
      "\n",
      "----\n"
     ]
    }
   ],
   "source": [
    "t = crossproduct(editor.template(\n",
    "    {\n",
    "        'contexts': [\n",
    "            '{male} and {female} are friends. His mom is {a:prof}.',\n",
    "            '{female} and {male} are friends. His mom is {a:prof}.',\n",
    "        ],\n",
    "        'qas': [\n",
    "            (\n",
    "                'Whose mom is {a:prof}?',\n",
    "                '{male}'\n",
    "            ), \n",
    "        ]\n",
    "        \n",
    "    },\n",
    "    save=True,\n",
    "    prof=professions,\n",
    "    remove_duplicates=True,\n",
    "    nsamples=250,\n",
    "    ))\n",
    "t += crossproduct(editor.template(\n",
    "    {\n",
    "        'contexts': [\n",
    "            '{male} and {female} are friends. Her mom is {a:prof}.',\n",
    "            '{female} and {male} are friends. Her mom is {a:prof}.',\n",
    "        ],\n",
    "        'qas': [\n",
    "            (\n",
    "                'Whose mom is {a:prof}?',\n",
    "                '{female}'\n",
    "            ), \n",
    "        ]\n",
    "        \n",
    "    },\n",
    "    save=True,\n",
    "    prof=professions,\n",
    "    remove_duplicates=True,\n",
    "    nsamples=250,\n",
    "    ))\n",
    "\n",
    "name = 'Basic coref, his / her'\n",
    "test = MFT(**t, expect=expect_squad, name=name, description='', capability='Coref')\n",
    "test.run(new_pp)\n",
    "test.summary(n=3, format_example_fn=format_squad_with_context)\n",
    "suite.add(test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Former, latter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting 1900 examples\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bbefd0be10154d43ae8c70930911fae8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=238.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Test cases:      475\n",
      "Fails (rate):    475 (100.0%)\n",
      "\n",
      "Example fails:\n",
      "C: Jason and Mark are friends. The former is an interpreter.\n",
      "Q: Who is an interpreter?\n",
      "A: Jason\n",
      "P: Mark\n",
      "\n",
      "C: Jason and Mark are friends. The former is an interpreter and the latter is an investigator.\n",
      "Q: Who is an interpreter?\n",
      "A: Jason\n",
      "P: Mark\n",
      "\n",
      "\n",
      "----\n",
      "C: Lauren and Andrew are friends. The former is an educator.\n",
      "Q: Who is an educator?\n",
      "A: Lauren\n",
      "P: Andrew\n",
      "\n",
      "C: Lauren and Andrew are friends. The former is an educator and the latter is an economist.\n",
      "Q: Who is an educator?\n",
      "A: Lauren\n",
      "P: Andrew\n",
      "\n",
      "\n",
      "----\n",
      "C: Jonathan and William are friends. The former is a producer.\n",
      "Q: Who is a producer?\n",
      "A: Jonathan\n",
      "P: William\n",
      "\n",
      "C: Jonathan and William are friends. The former is a producer and the latter is an interpreter.\n",
      "Q: Who is a producer?\n",
      "A: Jonathan\n",
      "P: Jonathan and William\n",
      "\n",
      "\n",
      "----\n"
     ]
    }
   ],
   "source": [
    "t = crossproduct(editor.template(\n",
    "    {\n",
    "        'contexts': [\n",
    "            '{first_name} and {first_name2} are friends. The former is {a:prof1}.',\n",
    "            '{first_name2} and {first_name} are friends. The latter is {a:prof1}.',\n",
    "            '{first_name} and {first_name2} are friends. The former is {a:prof1} and the latter is {a:prof2}.',\n",
    "            '{first_name2} and {first_name} are friends. The former is {a:prof2} and the latter is {a:prof1}.',\n",
    "        ],\n",
    "        'qas': [\n",
    "            (\n",
    "                'Who is {a:prof1}?',\n",
    "                '{first_name}'\n",
    "            ), \n",
    "        ]\n",
    "        \n",
    "    },\n",
    "    prof=professions,\n",
    "    remove_duplicates=True,\n",
    "    nsamples=500,\n",
    "    save=True\n",
    "    ))\n",
    "name = 'Former / Latter'\n",
    "test = MFT(**t, expect=expect_squad, name=name, description='', capability='Coref')\n",
    "test.run(new_pp)\n",
    "test.summary(n=3, format_example_fn=format_squad_with_context)\n",
    "suite.add(test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SRL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting 1988 examples\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "90c97edf491b49fd88926844ace2b882",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=249.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Test cases:      497\n",
      "Fails (rate):    302 (60.8%)\n",
      "\n",
      "Example fails:\n",
      "C: Ethan attacks Alyssa.\n",
      "Q: Who is attacked?\n",
      "A: Alyssa\n",
      "P: Ethan\n",
      "\n",
      "\n",
      "----\n",
      "C: Lisa deserves Megan.\n",
      "Q: Who is deserved?\n",
      "A: Megan\n",
      "P: Lisa\n",
      "\n",
      "C: Megan is deserved by Lisa.\n",
      "Q: Who deserves?\n",
      "A: Lisa\n",
      "P: Megan\n",
      "\n",
      "\n",
      "----\n",
      "C: Jennifer accepts Michelle.\n",
      "Q: Who is accepted?\n",
      "A: Michelle\n",
      "P: Jennifer accepts Michelle\n",
      "\n",
      "\n",
      "----\n"
     ]
    }
   ],
   "source": [
    "import pattern\n",
    "import pattern.en\n",
    "pverb = ['love', 'hate', 'like', 'remember', 'recognize', 'trust', 'deserve', 'understand', 'blame', 'dislike', 'prefer', 'follow', 'notice', 'hurt', 'bother', 'support', 'believe', 'accept', 'attack']\n",
    "a = pattern.en.tenses('loves')[0]\n",
    "b = pattern.en.tenses('stolen')[0]\n",
    "pverb = [(pattern.en.conjugate(v, *a), pattern.en.conjugate(v, *b)) for v in pverb]\n",
    "\n",
    "t = crossproduct(editor.template(\n",
    "    {\n",
    "        'contexts': [\n",
    "            '{first_name} {v[0]} {first_name2}.',\n",
    "            '{first_name2} is {v[1]} by {first_name}.',\n",
    "        ],\n",
    "        'qas': [\n",
    "            (\n",
    "                'Who {v[0]}?',\n",
    "                '{first_name}'\n",
    "            ), \n",
    "            (\n",
    "                'Who is {v[1]}?',\n",
    "                '{first_name2}'\n",
    "            ), \n",
    "        ]\n",
    "        \n",
    "    },\n",
    "    v=pverb,\n",
    "    remove_duplicates=True,\n",
    "    nsamples=500,\n",
    "    ))\n",
    "name = 'Agent / object distinction'\n",
    "test = MFT(**t, expect=expect_squad, name=name, description='', capability='SRL')\n",
    "test.run(new_pp)\n",
    "test.summary(n=3, format_example_fn=format_squad_with_context)\n",
    "suite.add(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting 7728 examples\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "74d0c715eb0d47b3a16d1a2f4d7a0609",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=966.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Test cases:      483\n",
      "Fails (rate):    462 (95.7%)\n",
      "\n",
      "Example fails:\n",
      "C: Jonathan is attacked by Jason. Jonathan attacks Joshua.\n",
      "Q: Who is attacked by Jonathan?\n",
      "A: Joshua\n",
      "P: Jason\n",
      "\n",
      "\n",
      "----\n",
      "C: Anna is followed by Sarah. Anna follows Amanda.\n",
      "Q: Who follows Amanda?\n",
      "A: Anna\n",
      "P: Sarah. Anna\n",
      "\n",
      "C: Anna is followed by Sarah. Anna follows Amanda.\n",
      "Q: Who is followed by Anna?\n",
      "A: Amanda\n",
      "P: Sarah\n",
      "\n",
      "C: Anna is followed by Sarah. Amanda is followed by Anna.\n",
      "Q: Who is followed by Anna?\n",
      "A: Amanda\n",
      "P: Sarah. Amanda\n",
      "\n",
      "\n",
      "----\n",
      "C: Emma prefers Alexander. Noah is preferred by Alexander.\n",
      "Q: Who prefers Noah?\n",
      "A: Alexander\n",
      "P: Emma\n",
      "\n",
      "C: Emma prefers Alexander. Noah is preferred by Alexander.\n",
      "Q: Who is preferred by Emma?\n",
      "A: Alexander\n",
      "P: Noah\n",
      "\n",
      "C: Alexander is preferred by Emma. Alexander prefers Noah.\n",
      "Q: Who is preferred by Alexander?\n",
      "A: Noah\n",
      "P: Emma\n",
      "\n",
      "\n",
      "----\n"
     ]
    }
   ],
   "source": [
    "t = crossproduct(editor.template(\n",
    "    {\n",
    "        'contexts': [\n",
    "            '{first_name} {v[0]} {first_name2}. {first_name2} {v[0]} {first_name3}.',\n",
    "            '{first_name} {v[0]} {first_name2}. {first_name3} is {v[1]} by {first_name2}.',\n",
    "            '{first_name2} is {v[1]} by {first_name}. {first_name2} {v[0]} {first_name3}.',\n",
    "            '{first_name2} is {v[1]} by {first_name}. {first_name3} is {v[1]} by {first_name2}.',\n",
    "        ],\n",
    "        'qas': [\n",
    "            (\n",
    "                'Who {v[0]} {first_name2}?',\n",
    "                '{first_name}'\n",
    "            ), \n",
    "            (\n",
    "                'Who {v[0]} {first_name3}?',\n",
    "                '{first_name2}'\n",
    "            ), \n",
    "            (\n",
    "                'Who is {v[1]} by {first_name}?',\n",
    "                '{first_name2}'\n",
    "            ), \n",
    "            (\n",
    "                'Who is {v[1]} by {first_name2}?',\n",
    "                '{first_name3}'\n",
    "            ), \n",
    "        ]\n",
    "        \n",
    "    },\n",
    "    save=True,\n",
    "    v=pverb,\n",
    "    remove_duplicates=True,\n",
    "    nsamples=500,\n",
    "    ))\n",
    "name = 'Agent / object distinction with 3 agents'\n",
    "test = MFT(**t, expect=expect_squad, name=name, description='', capability='SRL')\n",
    "test.run(new_pp)\n",
    "test.summary(n=3, format_example_fn=format_squad_with_context)\n",
    "suite.add(test)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = '/home/marcotcr/work/checklist/release_data/squad/squad_suite.pkl'\n",
    "suite.save(path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Vocabulary\n",
      "\n",
      "A is COMP than B. Who is more / less COMP?\n",
      "Test cases:      494\n",
      "Fails (rate):    99 (20.0%)\n",
      "\n",
      "Example fails:\n",
      "C: Shannon is darker than Amy.\n",
      "Q: Who is less dark?\n",
      "A: Amy\n",
      "P: Shannon\n",
      "\n",
      "\n",
      "----\n",
      "C: Alexander is shorter than Jeffrey.\n",
      "Q: Who is shorter?\n",
      "A: Alexander\n",
      "P: Jeffrey\n",
      "\n",
      "\n",
      "----\n",
      "C: Jeremy is shorter than Sarah.\n",
      "Q: Who is less short?\n",
      "A: Sarah\n",
      "P: Jeremy\n",
      "\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "Intensifiers (very, super, extremely) and reducers (somewhat, kinda, etc)?\n",
      "Test cases:      497\n",
      "Fails (rate):    454 (91.3%)\n",
      "\n",
      "Example fails:\n",
      "C: Sean is super cautious about the project. Jordan is cautious about the project.\n",
      "Q: Who is most cautious about the project?\n",
      "A: Sean\n",
      "P: Jordan\n",
      "\n",
      "C: Jordan is cautious about the project. Sean is super cautious about the project.\n",
      "Q: Who is most cautious about the project?\n",
      "A: Sean\n",
      "P: Jordan\n",
      "\n",
      "C: Jordan is cautious about the project. Sean is super cautious about the project.\n",
      "Q: Who is least cautious about the project?\n",
      "A: Jordan\n",
      "P: Sean\n",
      "\n",
      "\n",
      "----\n",
      "C: Tiffany is ambitious about the project. Steven is super ambitious about the project.\n",
      "Q: Who is least ambitious about the project?\n",
      "A: Tiffany\n",
      "P: Steven\n",
      "\n",
      "C: Tiffany is somewhat ambitious about the project. Steven is ambitious about the project.\n",
      "Q: Who is least ambitious about the project?\n",
      "A: Tiffany\n",
      "P: Steven\n",
      "\n",
      "C: Tiffany is somewhat ambitious about the project. Steven is super ambitious about the project.\n",
      "Q: Who is least ambitious about the project?\n",
      "A: Tiffany\n",
      "P: Steven\n",
      "\n",
      "\n",
      "----\n",
      "C: Sean is particular about the project. Nicole is particularly particular about the project.\n",
      "Q: Who is least particular about the project?\n",
      "A: Sean\n",
      "P: Nicole\n",
      "\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Taxonomy\n",
      "\n",
      "size, shape, age, color\n",
      "Test cases:      500\n",
      "Fails (rate):    412 (82.4%)\n",
      "\n",
      "Example fails:\n",
      "C: There is a figure in the room. The figure is tiny and old.\n",
      "Q: What size is the figure?\n",
      "A: tiny\n",
      "P: tiny and old\n",
      "\n",
      "\n",
      "----\n",
      "C: There is an object in the room. The object is round and red.\n",
      "Q: What shape is the object?\n",
      "A: round\n",
      "P: round and red\n",
      "\n",
      "\n",
      "----\n",
      "C: There is a small white box in the room.\n",
      "Q: What size is the box?\n",
      "A: small\n",
      "P: white\n",
      "\n",
      "C: There is a box in the room. The box is small and white.\n",
      "Q: What size is the box?\n",
      "A: small\n",
      "P: small and white\n",
      "\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "Profession vs nationality\n",
      "Test cases:      500\n",
      "Fails (rate):    247 (49.4%)\n",
      "\n",
      "Example fails:\n",
      "C: Eric is a Chinese investor.\n",
      "Q: What is Eric's job?\n",
      "A: investor\n",
      "P: Chinese investor\n",
      "\n",
      "\n",
      "----\n",
      "C: Shannon is an American interpreter.\n",
      "Q: What is Shannon's job?\n",
      "A: interpreter\n",
      "P: American interpreter\n",
      "\n",
      "\n",
      "----\n",
      "C: Amber is a Nigerian investor.\n",
      "Q: What is Amber's job?\n",
      "A: investor\n",
      "P: Nigerian investor\n",
      "\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "Animal vs Vehicle\n",
      "Test cases:      500\n",
      "Fails (rate):    128 (25.6%)\n",
      "\n",
      "Example fails:\n",
      "C: Ethan has a hamster and a firetruck.\n",
      "Q: What vehicle does Ethan have?\n",
      "A: firetruck\n",
      "P: a hamster and a firetruck\n",
      "\n",
      "C: Ethan has a firetruck and a hamster.\n",
      "Q: What animal does Ethan have?\n",
      "A: hamster\n",
      "P: a firetruck and a hamster\n",
      "\n",
      "\n",
      "----\n",
      "C: Hannah has a tractor and a cat.\n",
      "Q: What animal does Hannah have?\n",
      "A: cat\n",
      "P: a tractor and a cat\n",
      "\n",
      "\n",
      "----\n",
      "C: Noah has a duck and a firetruck.\n",
      "Q: What vehicle does Noah have?\n",
      "A: firetruck\n",
      "P: a duck and a firetruck\n",
      "\n",
      "C: Noah has a firetruck and a duck.\n",
      "Q: What animal does Noah have?\n",
      "A: duck\n",
      "P: a firetruck and a duck\n",
      "\n",
      "C: Noah has a firetruck and a duck.\n",
      "Q: What vehicle does Noah have?\n",
      "A: firetruck\n",
      "P: a firetruck and a duck\n",
      "\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "Animal vs Vehicle v2\n",
      "Test cases:      496\n",
      "Fails (rate):    130 (26.2%)\n",
      "\n",
      "Example fails:\n",
      "C: Tyler bought a serpent. Mark bought a train.\n",
      "Q: Who bought a vehicle?\n",
      "A: Mark\n",
      "P: Tyler bought a serpent. Mark\n",
      "\n",
      "\n",
      "----\n",
      "C: Dylan bought a bull. Justin bought a tractor.\n",
      "Q: Who bought a vehicle?\n",
      "A: Justin\n",
      "P: Dylan\n",
      "\n",
      "C: Justin bought a tractor. Dylan bought a bull.\n",
      "Q: Who bought an animal?\n",
      "A: Dylan\n",
      "P: Justin bought a tractor. Dylan\n",
      "\n",
      "\n",
      "----\n",
      "C: Michael bought a lizard. Brittany bought a train.\n",
      "Q: Who bought a vehicle?\n",
      "A: Brittany\n",
      "P: Michael\n",
      "\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "Synonyms\n",
      "Test cases:      447\n",
      "Fails (rate):    0 (0.0%)\n",
      "\n",
      "\n",
      "A is COMP than B. Who is antonym(COMP)? B\n",
      "Test cases:      496\n",
      "Fails (rate):    334 (67.3%)\n",
      "\n",
      "Example fails:\n",
      "C: Alexis is bigger than Alexander.\n",
      "Q: Who is smaller?\n",
      "A: Alexander\n",
      "P: Alexis\n",
      "\n",
      "\n",
      "----\n",
      "C: Jacob is worse than Taylor.\n",
      "Q: Who is better?\n",
      "A: Taylor\n",
      "P: Jacob\n",
      "\n",
      "\n",
      "----\n",
      "C: Nicholas is smarter than Sarah.\n",
      "Q: Who is dumber?\n",
      "A: Sarah\n",
      "P: Nicholas is smarter than Sarah\n",
      "\n",
      "C: Sarah is dumber than Nicholas.\n",
      "Q: Who is smarter?\n",
      "A: Nicholas\n",
      "P: Sarah\n",
      "\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "A is more X than B. Who is more antonym(X)? B. Who is less X? B. Who is more X? A. Who is less antonym(X)? A.\n",
      "Test cases:      491\n",
      "Fails (rate):    491 (100.0%)\n",
      "\n",
      "Example fails:\n",
      "C: Aaron is more bad than Christina.\n",
      "Q: Who is more good?\n",
      "A: Christina\n",
      "P: Aaron\n",
      "\n",
      "C: Aaron is more bad than Christina.\n",
      "Q: Who is less good?\n",
      "A: Aaron\n",
      "P: Christina\n",
      "\n",
      "C: Christina is more good than Aaron.\n",
      "Q: Who is less bad?\n",
      "A: Christina\n",
      "P: Aaron\n",
      "\n",
      "\n",
      "----\n",
      "C: Laura is more religious than Angela.\n",
      "Q: Who is more secular?\n",
      "A: Angela\n",
      "P: Laura\n",
      "\n",
      "C: Laura is more religious than Angela.\n",
      "Q: Who is less secular?\n",
      "A: Laura\n",
      "P: Angela\n",
      "\n",
      "C: Angela is more secular than Laura.\n",
      "Q: Who is more religious?\n",
      "A: Laura\n",
      "P: Angela\n",
      "\n",
      "\n",
      "----\n",
      "C: Angela is more insecure than Amber.\n",
      "Q: Who is more secure?\n",
      "A: Amber\n",
      "P: Angela\n",
      "\n",
      "C: Amber is more secure than Angela.\n",
      "Q: Who is less insecure?\n",
      "A: Amber\n",
      "P: Angela\n",
      "\n",
      "C: Angela is less secure than Amber.\n",
      "Q: Who is more insecure?\n",
      "A: Angela\n",
      "P: Amber\n",
      "\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Robustness\n",
      "\n",
      "Question typo\n",
      "Test cases:      500\n",
      "Fails (rate):    58 (11.6%)\n",
      "\n",
      "Example fails:\n",
      "C: Since the 1980s, Lutheran Church denominations have repudiated Martin Luther's statements against the Jews and have rejected the use of them to incite hatred against Lutherans. Strommen et al.'s 1970 survey of 4,745 North American Lutherans aged 15–65 found that, compared to the other minority groups under consideration, Lutherans were the least prejudiced toward Jews. Nevertheless, Professor Richard (Dick) Geary, former Professor of Modern History at the University of Nottingham, England, and the author of Hitler and Nazism (Routledge 1993), wrote in the journal History Today an article on who voted for the Nazis in elections held from 1928-1933, where he claimed that from his research he found that the Nazis gained disproportionately more votes from Protestant than Catholic areas of Germany.\n",
      "Q: What did a survey of North American Lutherans find that Lutherans felt about Jews compared to other minority groups?\n",
      "P: least prejudiced\n",
      "\n",
      "C: Since the 1980s, Lutheran Church denominations have repudiated Martin Luther's statements against the Jews and have rejected the use of them to incite hatred against Lutherans. Strommen et al.'s 1970 survey of 4,745 North American Lutherans aged 15–65 found that, compared to the other minority groups under consideration, Lutherans were the least prejudiced toward Jews. Nevertheless, Professor Richard (Dick) Geary, former Professor of Modern History at the University of Nottingham, England, and the author of Hitler and Nazism (Routledge 1993), wrote in the journal History Today an article on who voted for the Nazis in elections held from 1928-1933, where he claimed that from his research he found that the Nazis gained disproportionately more votes from Protestant than Catholic areas of Germany.\n",
      "Q: What did a survey of North American Lutherans find that Lutherans fel tabout Jews compared to other minority groups?\n",
      "P: least prejudiced toward Jews\n",
      "\n",
      "\n",
      "----\n",
      "C: Originating as the Jama'at al-Tawhid wal-Jihad in 1999, it pledged allegiance to al-Qaeda in 2004, participated in the Iraqi insurgency that followed the March 2003 invasion of Iraq by Western forces, joined the fight in the Syrian Civil War beginning in March 2011, and was expelled from al-Qaeda in early 2014, (which complained of its failure to consult and \"notorious intransigence\"). The group gained prominence after it drove Iraqi government forces out of key cities in western Iraq in a 2014 offensive. The group is adept at social media, posting Internet videos of beheadings of soldiers, civilians, journalists and aid workers, and is known for its destruction of cultural heritage sites. The United Nations has held ISIL responsible for human rights abuses and war crimes, and Amnesty International has reported ethnic cleansing by the group on a \"historic scale\". The group has been designated a terrorist organisation by the United Nations, the European Union and member states, the United States, India, Indonesia, Turkey, Saudi Arabia, Syria and other countries.\n",
      "Q: What has the United Nations designed ISIL?\n",
      "P: human rights abuses and war crimes\n",
      "\n",
      "C: Originating as the Jama'at al-Tawhid wal-Jihad in 1999, it pledged allegiance to al-Qaeda in 2004, participated in the Iraqi insurgency that followed the March 2003 invasion of Iraq by Western forces, joined the fight in the Syrian Civil War beginning in March 2011, and was expelled from al-Qaeda in early 2014, (which complained of its failure to consult and \"notorious intransigence\"). The group gained prominence after it drove Iraqi government forces out of key cities in western Iraq in a 2014 offensive. The group is adept at social media, posting Internet videos of beheadings of soldiers, civilians, journalists and aid workers, and is known for its destruction of cultural heritage sites. The United Nations has held ISIL responsible for human rights abuses and war crimes, and Amnesty International has reported ethnic cleansing by the group on a \"historic scale\". The group has been designated a terrorist organisation by the United Nations, the European Union and member states, the United States, India, Indonesia, Turkey, Saudi Arabia, Syria and other countries.\n",
      "Q: What has the United Nations desgined ISIL?\n",
      "P: terrorist organisation\n",
      "\n",
      "\n",
      "----\n",
      "C: On December 28, 2015, ESPN Deportes announced that they had reached an agreement with CBS and the NFL to be the exclusive Spanish-language broadcaster of the game, marking the third dedicated Spanish-language broadcast of the Super Bowl. Unlike NBC and Fox, CBS does not have a Spanish-language outlet of its own that could broadcast the game (though per league policy, a separate Spanish play-by-play call was carried on CBS's second audio program channel for over-the-air viewers). The game was called by ESPN Deportes' Monday Night Football commentary crew of Alvaro Martin and Raul Allegre, and sideline reporter John Sutcliffe. ESPN Deportes broadcast pre-game and post-game coverage, while Martin, Allegre, and Sutcliffe contributed English-language reports for ESPN's SportsCenter and Mike & Mike.\n",
      "Q: Which network broadcast the game in Spanish?\n",
      "P: ESPN Deportes\n",
      "\n",
      "C: On December 28, 2015, ESPN Deportes announced that they had reached an agreement with CBS and the NFL to be the exclusive Spanish-language broadcaster of the game, marking the third dedicated Spanish-language broadcast of the Super Bowl. Unlike NBC and Fox, CBS does not have a Spanish-language outlet of its own that could broadcast the game (though per league policy, a separate Spanish play-by-play call was carried on CBS's second audio program channel for over-the-air viewers). The game was called by ESPN Deportes' Monday Night Football commentary crew of Alvaro Martin and Raul Allegre, and sideline reporter John Sutcliffe. ESPN Deportes broadcast pre-game and post-game coverage, while Martin, Allegre, and Sutcliffe contributed English-language reports for ESPN's SportsCenter and Mike & Mike.\n",
      "Q: Which network boradcast the game in Spanish?\n",
      "P: ESPN\n",
      "\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "Question contractions\n",
      "Test cases:      500\n",
      "Fails (rate):    17 (3.4%)\n",
      "\n",
      "Example fails:\n",
      "C: The early United States expressed its opposition to Imperialism, at least in a form distinct from its own Manifest Destiny, through policies such as the Monroe Doctrine. However, beginning in the late 19th and early 20th century, policies such as Theodore Roosevelt’s interventionism in Central America and Woodrow Wilson’s mission to \"make the world safe for democracy\" changed all this. They were often backed by military force, but were more often affected from behind the scenes. This is consistent with the general notion of hegemony and imperium of historical empires. In 1898, Americans who opposed imperialism created the Anti-Imperialist League to oppose the US annexation of the Philippines and Cuba. One year later, a war erupted in the Philippines causing business, labor and government leaders in the US to condemn America's occupation in the Philippines as they also denounced them for causing the deaths of many Filipinos. American foreign policy was denounced as a \"racket\" by Smedley Butler, an American general. He said, \"Looking back on it, I might have given Al Capone a few hints. The best he could do was to operate his racket in three districts. I operated on three continents\".\n",
      "Q: How did the United States plan to subdue imperialistic tendencies? \n",
      "P: make the world safe for democracy\n",
      "\n",
      "C: The early United States expressed its opposition to Imperialism, at least in a form distinct from its own Manifest Destiny, through policies such as the Monroe Doctrine. However, beginning in the late 19th and early 20th century, policies such as Theodore Roosevelt’s interventionism in Central America and Woodrow Wilson’s mission to \"make the world safe for democracy\" changed all this. They were often backed by military force, but were more often affected from behind the scenes. This is consistent with the general notion of hegemony and imperium of historical empires. In 1898, Americans who opposed imperialism created the Anti-Imperialist League to oppose the US annexation of the Philippines and Cuba. One year later, a war erupted in the Philippines causing business, labor and government leaders in the US to condemn America's occupation in the Philippines as they also denounced them for causing the deaths of many Filipinos. American foreign policy was denounced as a \"racket\" by Smedley Butler, an American general. He said, \"Looking back on it, I might have given Al Capone a few hints. The best he could do was to operate his racket in three districts. I operated on three continents\".\n",
      "Q: How'd the United States plan to subdue imperialistic tendencies? \n",
      "P: make the world safe for democracy\"\n",
      "\n",
      "\n",
      "----\n",
      "C: Religious and spiritual teachers, such as gurus, mullahs, rabbis, pastors/youth pastors and lamas, may teach religious texts such as the Quran, Torah or Bible.\n",
      "Q: What is another general name for a religious teacher?\n",
      "P: spiritual\n",
      "\n",
      "C: Religious and spiritual teachers, such as gurus, mullahs, rabbis, pastors/youth pastors and lamas, may teach religious texts such as the Quran, Torah or Bible.\n",
      "Q: What's another general name for a religious teacher?\n",
      "P: gurus\n",
      "\n",
      "\n",
      "----\n",
      "C: It is a common misconception to ascribe the stiffness and rigidity of solid matter to the repulsion of like charges under the influence of the electromagnetic force. However, these characteristics actually result from the Pauli exclusion principle.[citation needed] Since electrons are fermions, they cannot occupy the same quantum mechanical state as other electrons. When the electrons in a material are densely packed together, there are not enough lower energy quantum mechanical states for them all, so some of them must be in higher energy states. This means that it takes energy to pack them together. While this effect is manifested macroscopically as a structural force, it is technically only the result of the existence of a finite set of electron states.\n",
      "Q: What is often misunderstood as the cause of matter rigidity?\n",
      "P: repulsion of like charges\n",
      "\n",
      "C: It is a common misconception to ascribe the stiffness and rigidity of solid matter to the repulsion of like charges under the influence of the electromagnetic force. However, these characteristics actually result from the Pauli exclusion principle.[citation needed] Since electrons are fermions, they cannot occupy the same quantum mechanical state as other electrons. When the electrons in a material are densely packed together, there are not enough lower energy quantum mechanical states for them all, so some of them must be in higher energy states. This means that it takes energy to pack them together. While this effect is manifested macroscopically as a structural force, it is technically only the result of the existence of a finite set of electron states.\n",
      "Q: What's often misunderstood as the cause of matter rigidity?\n",
      "P: repulsion of like charges under the influence of the electromagnetic force\n",
      "\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "Add random sentence to context\n",
      "Test cases:      500\n",
      "Fails (rate):    49 (9.8%)\n",
      "\n",
      "Example fails:\n",
      "C: Luther's hymns inspired composers to write music. Johann Sebastian Bach included several verses as chorales in his cantatas and based chorale cantatas entirely on them, namely Christ lag in Todes Banden, BWV 4, as early as possibly 1707, in his second annual cycle (1724 to 1725) Ach Gott, vom Himmel sieh darein, BWV 2, Christ unser Herr zum Jordan kam, BWV 7, Nun komm, der Heiden Heiland, BWV 62, Gelobet seist du, Jesu Christ, BWV 91, and Aus tiefer Not schrei ich zu dir, BWV 38, later Ein feste Burg ist unser Gott, BWV 80, and in 1735 Wär Gott nicht mit uns diese Zeit, BWV 14.\n",
      "Q: When was the last use by Bach of Luther's work?\n",
      "P: 1735\n",
      "\n",
      "C: Luther's hymns inspired composers to write music. Johann Sebastian Bach included several verses as chorales in his cantatas and based chorale cantatas entirely on them, namely Christ lag in Todes Banden, BWV 4, as early as possibly 1707, in his second annual cycle (1724 to 1725) Ach Gott, vom Himmel sieh darein, BWV 2, Christ unser Herr zum Jordan kam, BWV 7, Nun komm, der Heiden Heiland, BWV 62, Gelobet seist du, Jesu Christ, BWV 91, and Aus tiefer Not schrei ich zu dir, BWV 38, later Ein feste Burg ist unser Gott, BWV 80, and in 1735 Wär Gott nicht mit uns diese Zeit, BWV 14.Priestley published his findings in 1775 in a paper titled \"An Account of Further Discoveries in Air\" which was included in the second volume of his book titled Experiments and Observations on Different Kinds of Air. \n",
      "Q: When was the last use by Bach of Luther's work?\n",
      "P: 1724 to 1725\n",
      "\n",
      "C: Priestley published his findings in 1775 in a paper titled \"An Account of Further Discoveries in Air\" which was included in the second volume of his book titled Experiments and Observations on Different Kinds of Air. Luther's hymns inspired composers to write music. Johann Sebastian Bach included several verses as chorales in his cantatas and based chorale cantatas entirely on them, namely Christ lag in Todes Banden, BWV 4, as early as possibly 1707, in his second annual cycle (1724 to 1725) Ach Gott, vom Himmel sieh darein, BWV 2, Christ unser Herr zum Jordan kam, BWV 7, Nun komm, der Heiden Heiland, BWV 62, Gelobet seist du, Jesu Christ, BWV 91, and Aus tiefer Not schrei ich zu dir, BWV 38, later Ein feste Burg ist unser Gott, BWV 80, and in 1735 Wär Gott nicht mit uns diese Zeit, BWV 14.\n",
      "Q: When was the last use by Bach of Luther's work?\n",
      "P: 1725\n",
      "\n",
      "\n",
      "----\n",
      "C: Subsequently, Californios (dissatisfied with inequitable taxes and land laws) and pro-slavery southerners in the lightly populated \"Cow Counties\" of southern California attempted three times in the 1850s to achieve a separate statehood or territorial status separate from Northern California. The last attempt, the Pico Act of 1859, was passed by the California State Legislature and signed by the State governor John B. Weller. It was approved overwhelmingly by nearly 75% of voters in the proposed Territory of Colorado. This territory was to include all the counties up to the then much larger Tulare County (that included what is now Kings, most of Kern, and part of Inyo counties) and San Luis Obispo County. The proposal was sent to Washington, D.C. with a strong advocate in Senator Milton Latham. However, the secession crisis following the election of Abraham Lincoln in 1860 led to the proposal never coming to a vote.\n",
      "Q: How many times did southern California attempt to achieve a separate statehood?\n",
      "P: three times\n",
      "\n",
      "C: However, at the same time, the UMC Judicial Council, in 2008, ruled that conferences can determine their own policy related to transgender pastors, and therefore some regional conferences have voted to recognize ordained transgender pastors. Subsequently, Californios (dissatisfied with inequitable taxes and land laws) and pro-slavery southerners in the lightly populated \"Cow Counties\" of southern California attempted three times in the 1850s to achieve a separate statehood or territorial status separate from Northern California. The last attempt, the Pico Act of 1859, was passed by the California State Legislature and signed by the State governor John B. Weller. It was approved overwhelmingly by nearly 75% of voters in the proposed Territory of Colorado. This territory was to include all the counties up to the then much larger Tulare County (that included what is now Kings, most of Kern, and part of Inyo counties) and San Luis Obispo County. The proposal was sent to Washington, D.C. with a strong advocate in Senator Milton Latham. However, the secession crisis following the election of Abraham Lincoln in 1860 led to the proposal never coming to a vote.\n",
      "Q: How many times did southern California attempt to achieve a separate statehood?\n",
      "P: three\n",
      "\n",
      "\n",
      "----\n",
      "C: OPEC soon lost its preeminent position, and in 1981, its production was surpassed by that of other countries. Additionally, its own member nations were divided. Saudi Arabia, trying to recover market share, increased production, pushing prices down, shrinking or eliminating profits for high-cost producers. The world price, which had peaked during the 1979 energy crisis at nearly $40 per barrel, decreased during the 1980s to less than $10 per barrel. Adjusted for inflation, oil briefly fell back to pre-1973 levels. This \"sale\" price was a windfall for oil-importing nations, both developing and developed.\n",
      "Q: Why did Saudi Arabia try to increase production, and reduce profits for high cost producers?\n",
      "P: trying to recover market share\n",
      "\n",
      "C: OPEC soon lost its preeminent position, and in 1981, its production was surpassed by that of other countries. Additionally, its own member nations were divided. Saudi Arabia, trying to recover market share, increased production, pushing prices down, shrinking or eliminating profits for high-cost producers. The world price, which had peaked during the 1979 energy crisis at nearly $40 per barrel, decreased during the 1980s to less than $10 per barrel. Adjusted for inflation, oil briefly fell back to pre-1973 levels. This \"sale\" price was a windfall for oil-importing nations, both developing and developed.Some of the most relevant Supreme Court case law on this is as follows: Runyon v. McCrary, 427 U.S. 160 (1976); Wisconsin v. Yoder, 406 U.S. 205 (1972); Pierce v. Society of Sisters, 268 U.S. 510 (1925); Meyer v. Nebraska, 262 U.S. 390 (1923). \n",
      "Q: Why did Saudi Arabia try to increase production, and reduce profits for high cost producers?\n",
      "P: to recover market share\n",
      "\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "NER\n",
      "\n",
      "Change name everywhere\n",
      "Test cases:      500\n",
      "Fails (rate):    29 (5.8%)\n",
      "\n",
      "Example fails:\n",
      "C: The first buildings of the University of Chicago campus, which make up what is now known as the Main Quadrangles, were part of a \"master plan\" conceived by two University of Chicago trustees and plotted by Chicago architect Henry Ives Cobb. The Main Quadrangles consist of six quadrangles, each surrounded by buildings, bordering one larger quadrangle. The buildings of the Main Quadrangles were designed by Cobb, Shepley, Rutan and Coolidge, Holabird & Roche, and other architectural firms in a mixture of the Victorian Gothic and Collegiate Gothic styles, patterned on the colleges of the University of Oxford. (Mitchell Tower, for example, is modeled after Oxford's Magdalen Tower, and the university Commons, Hutchinson Hall, replicates Christ Church Hall.)\n",
      "Q: Who helped designed the Main Quadrangles?\n",
      "P: Shepley, Rutan and Coolidge\n",
      "\n",
      "C: The first buildings of the University of Chicago campus, which make up what is now known as the Main Quadrangles, were part of a \"master plan\" conceived by two University of Chicago trustees and plotted by Chicago architect William Jones. The Main Quadrangles consist of six quadrangles, each surrounded by buildings, bordering one larger quadrangle. The buildings of the Main Quadrangles were designed by Cobb, Shepley, Rutan and Coolidge, Holabird & Roche, and other architectural firms in a mixture of the Victorian Gothic and Collegiate Gothic styles, patterned on the colleges of the University of Oxford. (Mitchell Tower, for example, is modeled after Oxford's Magdalen Tower, and the university Commons, Hutchinson Hall, replicates Christ Church Hall.)\n",
      "Q: Who helped designed the Main Quadrangles?\n",
      "P: Cobb, Shepley, Rutan and Coolidge\n",
      "\n",
      "C: The first buildings of the University of Chicago campus, which make up what is now known as the Main Quadrangles, were part of a \"master plan\" conceived by two University of Chicago trustees and plotted by Chicago architect Joshua Foster. The Main Quadrangles consist of six quadrangles, each surrounded by buildings, bordering one larger quadrangle. The buildings of the Main Quadrangles were designed by Cobb, Shepley, Rutan and Coolidge, Holabird & Roche, and other architectural firms in a mixture of the Victorian Gothic and Collegiate Gothic styles, patterned on the colleges of the University of Oxford. (Mitchell Tower, for example, is modeled after Oxford's Magdalen Tower, and the university Commons, Hutchinson Hall, replicates Christ Church Hall.)\n",
      "Q: Who helped designed the Main Quadrangles?\n",
      "P: Cobb, Shepley, Rutan and Coolidge\n",
      "\n",
      "\n",
      "----\n",
      "C: Islamists have asked the question, \"If Islam is a way of life, how can we say that those who want to live by its principles in legal, social, political, economic, and political spheres of life are not Muslims, but Islamists and believe in Islamism, not [just] Islam?\" Similarly, a writer for the International Crisis Group maintains that \"the conception of 'political Islam'\" is a creation of Americans to explain the Iranian Islamic Revolution and apolitical Islam was a historical fluke of the \"short-lived era of the heyday of secular Arab nationalism between 1945 and 1970\", and it is quietist/non-political Islam, not Islamism, that requires explanation.\n",
      "Q: What was apolitical Islam?\n",
      "P: a historical fluke of the \"short-lived era of the heyday of secular Arab nationalism between 1945 and 1970\"\n",
      "\n",
      "C: Islamists have asked the question, \"If Jeremiah is a way of life, how can we say that those who want to live by its principles in legal, social, political, economic, and political spheres of life are not Muslims, but Islamists and believe in Islamism, not [just] Jeremiah?\" Similarly, a writer for the International Crisis Group maintains that \"the conception of 'political Jeremiah'\" is a creation of Americans to explain the Iranian Islamic Revolution and apolitical Jeremiah was a historical fluke of the \"short-lived era of the heyday of secular Arab nationalism between 1945 and 1970\", and it is quietist/non-political Jeremiah, not Islamism, that requires explanation.\n",
      "Q: What was apolitical Jeremiah?\n",
      "P: a historical fluke of the \"short-lived era of the heyday of secular Arab nationalism between 1945 and 1970\n",
      "\n",
      "\n",
      "----\n",
      "C: On 6 November 1915, a Reuters news agency report from London had the 1915 Nobel Prize in Physics awarded to Thomas Edison and Nikola Tesla; however, on 15 November, a Reuters story from Stockholm stated the prize that year was being awarded to Sir William Henry Bragg and William Lawrence Bragg \"for their services in the analysis of crystal structure by means of X-rays.\":245 There were unsubstantiated rumors at the time that Tesla and/or Edison had refused the prize.:245 The Nobel Foundation said, \"Any rumor that a person has not been given a Nobel Prize because he has made known his intention to refuse the reward is ridiculous\"; a recipient could only decline a Nobel Prize after he is announced a winner.:245\n",
      "Q: What was the rumored reason Edison and Tesla were not awarded the prize?\n",
      "P: refuse the reward\n",
      "\n",
      "C: On 6 November 1915, a Reuters news agency report from London had the 1915 Nobel Prize in Physics awarded to Thomas Christian and Nikola Tesla; however, on 15 November, a Reuters story from Stockholm stated the prize that year was being awarded to Sir William Henry Bragg and William Lawrence Bragg \"for their services in the analysis of crystal structure by means of X-rays.\":245 There were unsubstantiated rumors at the time that Tesla and/or Christian had refused the prize.:245 The Nobel Foundation said, \"Any rumor that a person has not been given a Nobel Prize because he has made known his intention to refuse the reward is ridiculous\"; a recipient could only decline a Nobel Prize after he is announced a winner.:245\n",
      "Q: What was the rumored reason Christian and Tesla were not awarded the prize?\n",
      "P: he has made known his intention to refuse the reward\n",
      "\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "Change location everywhere\n",
      "Test cases:      500\n",
      "Fails (rate):    53 (10.6%)\n",
      "\n",
      "Example fails:\n",
      "C: Much of the city's tax base dissipated, leading to problems with funding education, sanitation, and traffic control within the city limits. In addition, residents in unincorporated suburbs had difficulty obtaining municipal services, such as sewage and building code enforcement. In 1958, a study recommended that the city of Jacksonville begin annexing outlying communities in order to create the needed tax base to improve services throughout the county. Voters outside the city limits rejected annexation plans in six referendums between 1960 and 1965.\n",
      "Q: What was the cause for the issues with city funding?\n",
      "P: Much of the city's tax base dissipated\n",
      "\n",
      "C: Much of the city's tax base dissipated, leading to problems with funding education, sanitation, and traffic control within the city limits. In addition, residents in unincorporated suburbs had difficulty obtaining municipal services, such as sewage and building code enforcement. In 1958, a study recommended that the city of Tacoma begin annexing outlying communities in order to create the needed tax base to improve services throughout the county. Voters outside the city limits rejected annexation plans in six referendums between 1960 and 1965.\n",
      "Q: What was the cause for the issues with city funding?\n",
      "P: tax base dissipated\n",
      "\n",
      "\n",
      "----\n",
      "C: For the next three hundred years, Scotland was directly governed by the Parliament of Great Britain and the subsequent Parliament of the United Kingdom, both seated at Westminster, and the lack of a Parliament of Scotland remained an important element in Scottish national identity. Suggestions for a 'devolved' Parliament were made before 1914, but were shelved due to the outbreak of the First World War. A sharp rise in nationalism in Scotland during the late 1960s fuelled demands for some form of home rule or complete independence, and in 1969 prompted the incumbent Labour government of Harold Wilson to set up the Kilbrandon Commission to consider the British constitution. One of the principal objectives of the commission was to examine ways of enabling more self-government for Scotland, within the unitary state of the United Kingdom. Kilbrandon published his report in 1973 recommending the establishment of a directly elected Scottish Assembly to legislate for the majority of domestic Scottish affairs.\n",
      "Q: What remained an important issue in Scottish national identity for many years?\n",
      "P: lack of a Parliament of Scotland\n",
      "\n",
      "C: For the next three hundred years, Scotland was directly governed by the Parliament of Great Britain and the subsequent Parliament of the United Kingdom, both seated at Chelsea, and the lack of a Parliament of Scotland remained an important element in Scottish national identity. Suggestions for a 'devolved' Parliament were made before 1914, but were shelved due to the outbreak of the First World War. A sharp rise in nationalism in Scotland during the late 1960s fuelled demands for some form of home rule or complete independence, and in 1969 prompted the incumbent Labour government of Harold Wilson to set up the Kilbrandon Commission to consider the British constitution. One of the principal objectives of the commission was to examine ways of enabling more self-government for Scotland, within the unitary state of the United Kingdom. Kilbrandon published his report in 1973 recommending the establishment of a directly elected Scottish Assembly to legislate for the majority of domestic Scottish affairs.\n",
      "Q: What remained an important issue in Scottish national identity for many years?\n",
      "P: the lack of a Parliament of Scotland\n",
      "\n",
      "C: For the next three hundred years, Scotland was directly governed by the Parliament of Great Britain and the subsequent Parliament of the United Kingdom, both seated at Stanton, and the lack of a Parliament of Scotland remained an important element in Scottish national identity. Suggestions for a 'devolved' Parliament were made before 1914, but were shelved due to the outbreak of the First World War. A sharp rise in nationalism in Scotland during the late 1960s fuelled demands for some form of home rule or complete independence, and in 1969 prompted the incumbent Labour government of Harold Wilson to set up the Kilbrandon Commission to consider the British constitution. One of the principal objectives of the commission was to examine ways of enabling more self-government for Scotland, within the unitary state of the United Kingdom. Kilbrandon published his report in 1973 recommending the establishment of a directly elected Scottish Assembly to legislate for the majority of domestic Scottish affairs.\n",
      "Q: What remained an important issue in Scottish national identity for many years?\n",
      "P: the lack of a Parliament of Scotland\n",
      "\n",
      "\n",
      "----\n",
      "C: The 8- and 10-county definitions are not used for the greater Southern California Megaregion, one of the 11 megaregions of the United States. The megaregion's area is more expansive, extending east into Las Vegas, Nevada, and south across the Mexican border into Tijuana.\n",
      "Q: What is the name of the state that the megaregion expands to in the east?\n",
      "P: Las Vegas, Nevada\n",
      "\n",
      "C: The 8- and 10-county definitions are not used for the greater Southern California Megaregion, one of the 11 megaregions of the United States. The megaregion's area is more expansive, extending east into Richmond, Nevada, and south across the Mexican border into Tijuana.\n",
      "Q: What is the name of the state that the megaregion expands to in the east?\n",
      "P: Richmond\n",
      "\n",
      "C: The 8- and 10-county definitions are not used for the greater Southern California Megaregion, one of the 11 megaregions of the United States. The megaregion's area is more expansive, extending east into San Marcos, Nevada, and south across the Mexican border into Tijuana.\n",
      "Q: What is the name of the state that the megaregion expands to in the east?\n",
      "P: San Marcos\n",
      "\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Fairness\n",
      "\n",
      "M/F failure rates should be similar for different professions\n",
      "Test cases:      2000\n",
      "Fails (rate):    925 (46.2%)\n",
      "\n",
      "Example fails:\n",
      "C: Avery is not a CEO, Robert is.\n",
      "Q: Who is a CEO?\n",
      "A: Robert\n",
      "P: Avery\n",
      "\n",
      "----\n",
      "C: Benjamin is not a doctor, Diana is.\n",
      "Q: Who is a doctor?\n",
      "A: Diana\n",
      "P: Benjamin\n",
      "\n",
      "----\n",
      "C: Dylan is not a CEO, Alicia is.\n",
      "Q: Who is a CEO?\n",
      "A: Alicia\n",
      "P: Dylan\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Temporal\n",
      "\n",
      "There was a change in profession\n",
      "Test cases:      482\n",
      "Fails (rate):    200 (41.5%)\n",
      "\n",
      "Example fails:\n",
      "C: Both Sarah and Katherine were attorneys, but there was a change in Katherine, who is now an auditor.\n",
      "Q: Who is an auditor?\n",
      "A: Katherine\n",
      "P: Katherine were attorneys, but there was a change in Katherine\n",
      "\n",
      "\n",
      "----\n",
      "C: Both Jordan and Erin were architects, but there was a change in Erin, who is now a historian.\n",
      "Q: Who is a historian?\n",
      "A: Erin\n",
      "P: Erin were architects, but there was a change in Erin\n",
      "\n",
      "\n",
      "----\n",
      "C: Both Andrea and Danielle were agents, but there was a change in Danielle, who is now a nurse.\n",
      "Q: Who is a nurse?\n",
      "A: Danielle\n",
      "P: Danielle were agents, but there was a change in Danielle\n",
      "\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "Understanding before / after -> first / last.\n",
      "Test cases:      496\n",
      "Fails (rate):    411 (82.9%)\n",
      "\n",
      "Example fails:\n",
      "C: Alyssa became a actor after Abigail did.\n",
      "Q: Who became a actor first?\n",
      "A: Abigail\n",
      "P: Alyssa\n",
      "\n",
      "C: Alyssa became a actor after Abigail did.\n",
      "Q: Who became a actor last?\n",
      "A: Alyssa\n",
      "P: Abigail\n",
      "\n",
      "\n",
      "----\n",
      "C: Samuel became a accountant before Anna did.\n",
      "Q: Who became a accountant last?\n",
      "A: Anna\n",
      "P: Samuel\n",
      "\n",
      "C: Anna became a accountant after Samuel did.\n",
      "Q: Who became a accountant first?\n",
      "A: Samuel\n",
      "P: Anna\n",
      "\n",
      "\n",
      "----\n",
      "C: Sara became a producer after Nicole did.\n",
      "Q: Who became a producer last?\n",
      "A: Sara\n",
      "P: Nicole\n",
      "\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Negation\n",
      "\n",
      "Negation in context, may or may not be in question\n",
      "Test cases:      499\n",
      "Fails (rate):    337 (67.5%)\n",
      "\n",
      "Example fails:\n",
      "C: Rebecca is not an actress. Jason is.\n",
      "Q: Who is an actress?\n",
      "A: Jason\n",
      "P: Rebecca\n",
      "\n",
      "\n",
      "----\n",
      "C: Jacob is not an actor. Angela is.\n",
      "Q: Who is an actor?\n",
      "A: Angela\n",
      "P: Jacob\n",
      "\n",
      "\n",
      "----\n",
      "C: Megan is not an author. Rebecca is.\n",
      "Q: Who is an author?\n",
      "A: Rebecca\n",
      "P: Megan\n",
      "\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "Negation in question only.\n",
      "Test cases:      481\n",
      "Fails (rate):    481 (100.0%)\n",
      "\n",
      "Example fails:\n",
      "C: Jessica is an attorney. Kevin is an executive.\n",
      "Q: Who is not an attorney?\n",
      "A: Kevin\n",
      "P: Jessica\n",
      "\n",
      "C: Jessica is an attorney. Kevin is an executive.\n",
      "Q: Who is not an executive?\n",
      "A: Jessica\n",
      "P: Kevin\n",
      "\n",
      "C: Kevin is an executive. Jessica is an attorney.\n",
      "Q: Who is not an attorney?\n",
      "A: Kevin\n",
      "P: Jessica\n",
      "\n",
      "\n",
      "----\n",
      "C: Kevin is an author. Alexis is an intern.\n",
      "Q: Who is not an author?\n",
      "A: Alexis\n",
      "P: Kevin\n",
      "\n",
      "C: Kevin is an author. Alexis is an intern.\n",
      "Q: Who is not an intern?\n",
      "A: Kevin\n",
      "P: Alexis\n",
      "\n",
      "C: Alexis is an intern. Kevin is an author.\n",
      "Q: Who is not an author?\n",
      "A: Alexis\n",
      "P: Kevin\n",
      "\n",
      "\n",
      "----\n",
      "C: Amy is a nurse. Ethan is an adviser.\n",
      "Q: Who is not a nurse?\n",
      "A: Ethan\n",
      "P: Amy\n",
      "\n",
      "C: Amy is a nurse. Ethan is an adviser.\n",
      "Q: Who is not an adviser?\n",
      "A: Amy\n",
      "P: Ethan\n",
      "\n",
      "C: Ethan is an adviser. Amy is a nurse.\n",
      "Q: Who is not a nurse?\n",
      "A: Ethan\n",
      "P: Amy\n",
      "\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Coref\n",
      "\n",
      "Basic coref, he / she\n",
      "Test cases:      477\n",
      "Fails (rate):    477 (100.0%)\n",
      "\n",
      "Example fails:\n",
      "C: Lisa and Kevin are friends. He is an executive, and she is an advisor.\n",
      "Q: Who is an advisor?\n",
      "A: Lisa\n",
      "P: Kevin\n",
      "\n",
      "C: Kevin and Lisa are friends. She is an advisor, and he is an executive.\n",
      "Q: Who is an executive?\n",
      "A: Kevin\n",
      "P: Lisa\n",
      "\n",
      "C: Lisa and Kevin are friends. She is an advisor, and he is an executive.\n",
      "Q: Who is an advisor?\n",
      "A: Lisa\n",
      "P: Kevin\n",
      "\n",
      "\n",
      "----\n",
      "C: Michelle and Scott are friends. He is an escort, and she is an accountant.\n",
      "Q: Who is an accountant?\n",
      "A: Michelle\n",
      "P: Scott\n",
      "\n",
      "C: Scott and Michelle are friends. She is an accountant, and he is an escort.\n",
      "Q: Who is an escort?\n",
      "A: Scott\n",
      "P: Michelle\n",
      "\n",
      "C: Michelle and Scott are friends. She is an accountant, and he is an escort.\n",
      "Q: Who is an accountant?\n",
      "A: Michelle\n",
      "P: Scott\n",
      "\n",
      "\n",
      "----\n",
      "C: Nathaniel and Hannah are friends. He is an auditor, and she is an advisor.\n",
      "Q: Who is an auditor?\n",
      "A: Nathaniel\n",
      "P: Nathaniel and Hannah\n",
      "\n",
      "C: Hannah and Nathaniel are friends. He is an auditor, and she is an advisor.\n",
      "Q: Who is an advisor?\n",
      "A: Hannah\n",
      "P: Nathaniel\n",
      "\n",
      "C: Nathaniel and Hannah are friends. She is an advisor, and he is an auditor.\n",
      "Q: Who is an auditor?\n",
      "A: Nathaniel\n",
      "P: Hannah\n",
      "\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "Basic coref, his / her\n",
      "Test cases:      500\n",
      "Fails (rate):    459 (91.8%)\n",
      "\n",
      "Example fails:\n",
      "C: Savannah and Jonathan are friends. Her mom is an activist.\n",
      "Q: Whose mom is an activist?\n",
      "A: Savannah\n",
      "P: Jonathan\n",
      "\n",
      "\n",
      "----\n",
      "C: Taylor and Brandon are friends. Her mom is an author.\n",
      "Q: Whose mom is an author?\n",
      "A: Taylor\n",
      "P: Brandon\n",
      "\n",
      "\n",
      "----\n",
      "C: Dustin and Sara are friends. His mom is an analyst.\n",
      "Q: Whose mom is an analyst?\n",
      "A: Dustin\n",
      "P: Dustin and Sara\n",
      "\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "Former / Latter\n",
      "Test cases:      475\n",
      "Fails (rate):    475 (100.0%)\n",
      "\n",
      "Example fails:\n",
      "C: Brandon and Amy are friends. The former is an advisor.\n",
      "Q: Who is an advisor?\n",
      "A: Brandon\n",
      "P: Amy\n",
      "\n",
      "\n",
      "----\n",
      "C: Melissa and Sarah are friends. The former is an economist.\n",
      "Q: Who is an economist?\n",
      "A: Melissa\n",
      "P: Sarah\n",
      "\n",
      "C: Melissa and Sarah are friends. The former is an economist and the latter is a photographer.\n",
      "Q: Who is an economist?\n",
      "A: Melissa\n",
      "P: Melissa and Sarah\n",
      "\n",
      "\n",
      "----\n",
      "C: Abigail and Joshua are friends. The former is an artist.\n",
      "Q: Who is an artist?\n",
      "A: Abigail\n",
      "P: Joshua\n",
      "\n",
      "C: Abigail and Joshua are friends. The former is an artist and the latter is an executive.\n",
      "Q: Who is an artist?\n",
      "A: Abigail\n",
      "P: Joshua\n",
      "\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "SRL\n",
      "\n",
      "Agent / object distinction\n",
      "Test cases:      497\n",
      "Fails (rate):    302 (60.8%)\n",
      "\n",
      "Example fails:\n",
      "C: Angela hurts Joshua.\n",
      "Q: Who hurts?\n",
      "A: Angela\n",
      "P: Angela hurts Joshua\n",
      "\n",
      "\n",
      "----\n",
      "C: Katherine supports Dylan.\n",
      "Q: Who supports?\n",
      "A: Katherine\n",
      "P: Katherine supports Dylan\n",
      "\n",
      "\n",
      "----\n",
      "C: Angela likes Tiffany.\n",
      "Q: Who likes?\n",
      "A: Angela\n",
      "P: Angela likes Tiffany\n",
      "\n",
      "C: Tiffany is liked by Angela.\n",
      "Q: Who likes?\n",
      "A: Angela\n",
      "P: Tiffany\n",
      "\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "Agent / object distinction with 3 agents\n",
      "Test cases:      483\n",
      "Fails (rate):    462 (95.7%)\n",
      "\n",
      "Example fails:\n",
      "C: Dylan is hated by Victoria. Dylan hates Erin.\n",
      "Q: Who is hated by Dylan?\n",
      "A: Erin\n",
      "P: Victoria\n",
      "\n",
      "\n",
      "----\n",
      "C: Sara understands Jose. Jessica is understood by Jose.\n",
      "Q: Who understands Jessica?\n",
      "A: Jose\n",
      "P: Sara\n",
      "\n",
      "C: Jose is understood by Sara. Jose understands Jessica.\n",
      "Q: Who is understood by Jose?\n",
      "A: Jessica\n",
      "P: Sara\n",
      "\n",
      "\n",
      "----\n",
      "C: Daniel attacks Kimberly. Jeremy is attacked by Kimberly.\n",
      "Q: Who attacks Jeremy?\n",
      "A: Kimberly\n",
      "P: Daniel\n",
      "\n",
      "C: Daniel attacks Kimberly. Jeremy is attacked by Kimberly.\n",
      "Q: Who is attacked by Daniel?\n",
      "A: Kimberly\n",
      "P: Kimberly. Jeremy\n",
      "\n",
      "C: Kimberly is attacked by Daniel. Kimberly attacks Jeremy.\n",
      "Q: Who is attacked by Kimberly?\n",
      "A: Jeremy\n",
      "P: Daniel. Kimberly attacks Jeremy\n",
      "\n",
      "\n",
      "----\n",
      "\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "suite.summary(n=3, format_example_fn=format_squad_with_context)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "profession    fail_men fail_women (count)\n",
      "CEO             16.9      96.6     (267)\n",
      "doctor          3.2      89.1     (247)\n",
      "secretary       60.5      4.0     (253)\n",
      "nurse           57.9      41.2     (233)\n"
     ]
    }
   ],
   "source": [
    "print_fair(suite.tests['M/F failure rates should be similar for different professions'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "format_fn = lambda x: json.dumps({'passage': x[0], 'question': x[1]})\n",
    "suite.to_raw_file('/tmp/squad.jsonl', format_fn=format_fn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "format_fn = lambda x: {'passage': x[0], 'question': x[1]}\n",
    "suite.to_raw_file('/tmp/squad.json', format_fn=format_fn, file_format='squad')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "checklist",
   "language": "python",
   "name": "checklist"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
